From 274673d9c6260e9122f2e771854742225d6b5a68 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Oct 2019 13:42:49 +0900 Subject: [PATCH 001/230] Add linear vesting to multisig License: MIT Signed-off-by: Jakub Sztandera --- chain/actors/actor_multisig.go | 35 ++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/chain/actors/actor_multisig.go b/chain/actors/actor_multisig.go index 1ddb9d6a3..85b1edf16 100644 --- a/chain/actors/actor_multisig.go +++ b/chain/actors/actor_multisig.go @@ -16,10 +16,29 @@ type MultiSigActorState struct { Required uint64 NextTxID uint64 + InitialBalance types.BigInt + StartingBlock uint64 + UnlockDuration uint64 + //TODO: make this map/sharray/whatever Transactions []MTransaction } +func (msas MultiSigActorState) canSpend(act *types.Actor, amnt types.BigInt, height uint64) bool { + if msas.UnlockDuration == 0 { + return true + } + + offset := height - msas.StartingBlock + if offset > msas.UnlockDuration { + return true + } + + minBalance := types.BigDiv(msas.InitialBalance, types.NewInt(msas.UnlockDuration)) + minBalance = types.BigMul(minBalance, types.NewInt(offset)) + return !minBalance.LessThan(types.BigSub(act.Balance, amnt)) +} + func (msas MultiSigActorState) isSigner(addr address.Address) bool { for _, s := range msas.Signers { if s == addr { @@ -90,8 +109,9 @@ func (msa MultiSigActor) Exports() []interface{} { } type MultiSigConstructorParams struct { - Signers []address.Address - Required uint64 + Signers []address.Address + Required uint64 + UnlockDuration uint64 } func (MultiSigActor) MultiSigConstructor(act *types.Actor, vmctx types.VMContext, @@ -99,7 +119,12 @@ func (MultiSigActor) MultiSigConstructor(act *types.Actor, vmctx types.VMContext self := &MultiSigActorState{ Signers: params.Signers, Required: params.Required, + + InitialBalance: vmctx.Message().Value, + UnlockDuration: params.UnlockDuration, + StartingBlock: vmctx.BlockHeight(), } + head, err := vmctx.Storage().Put(self) if err != nil { return nil, aerrors.Wrap(err, "could not put new head") @@ -179,6 +204,9 @@ func (msa MultiSigActor) Propose(act *types.Actor, vmctx types.VMContext, tx := self.getTransaction(txid) if self.Required == 1 { + if !self.canSpend(act, tx.Value, vmctx.BlockHeight()) { + return nil, aerrors.New(100, "transaction amount exceeds available") + } _, err := vmctx.Send(tx.To, tx.Method, tx.Value, tx.Params) if aerrors.IsFatal(err) { return nil, err @@ -221,6 +249,9 @@ func (msa MultiSigActor) Approve(act *types.Actor, vmctx types.VMContext, } tx.Approved = append(tx.Approved, vmctx.Message().From) if uint64(len(tx.Approved)) >= self.Required { + if !self.canSpend(act, tx.Value, vmctx.BlockHeight()) { + return nil, aerrors.New(100, "transaction amount exceeds available") + } _, err := vmctx.Send(tx.To, tx.Method, tx.Value, tx.Params) if aerrors.IsFatal(err) { return nil, err From 737f056c128e1e9333252a0f2add7c2840ae8631 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Oct 2019 14:04:19 +0900 Subject: [PATCH 002/230] Rebuild types License: MIT Signed-off-by: Jakub Sztandera --- chain/actors/cbor_gen.go | 67 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index d39a81af2..17f6f8128 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -1060,7 +1060,7 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{132}); err != nil { + if _, err := w.Write([]byte{135}); err != nil { return err } @@ -1084,6 +1084,21 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { return err } + // t.t.InitialBalance (types.BigInt) + if err := t.InitialBalance.MarshalCBOR(w); err != nil { + return err + } + + // t.t.StartingBlock (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.StartingBlock)); err != nil { + return err + } + + // t.t.UnlockDuration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + return err + } + // t.t.Transactions ([]actors.MTransaction) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Transactions)))); err != nil { return err @@ -1107,7 +1122,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 4 { + if extra != 7 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -1157,6 +1172,35 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.NextTxID = extra + // t.t.InitialBalance (types.BigInt) + + { + + if err := t.InitialBalance.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.StartingBlock (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.StartingBlock = extra + // t.t.UnlockDuration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.UnlockDuration = extra // t.t.Transactions ([]actors.MTransaction) maj, extra, err = cbg.CborReadHeader(br) @@ -1191,7 +1235,7 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{130}); err != nil { + if _, err := w.Write([]byte{131}); err != nil { return err } @@ -1209,6 +1253,11 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Required)); err != nil { return err } + + // t.t.UnlockDuration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + return err + } return nil } @@ -1223,7 +1272,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 2 { + if extra != 3 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -1263,6 +1312,16 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.Required = extra + // t.t.UnlockDuration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.UnlockDuration = extra return nil } From 92b145ec4d6d2b0a9d805194fa11c150eebf6d38 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 18 Oct 2019 16:08:19 +0900 Subject: [PATCH 003/230] Don't set fields if UnlockDuration is 0 License: MIT Signed-off-by: Jakub Sztandera --- chain/actors/actor_multisig.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/chain/actors/actor_multisig.go b/chain/actors/actor_multisig.go index 85b1edf16..b4a140544 100644 --- a/chain/actors/actor_multisig.go +++ b/chain/actors/actor_multisig.go @@ -119,10 +119,12 @@ func (MultiSigActor) MultiSigConstructor(act *types.Actor, vmctx types.VMContext self := &MultiSigActorState{ Signers: params.Signers, Required: params.Required, + } - InitialBalance: vmctx.Message().Value, - UnlockDuration: params.UnlockDuration, - StartingBlock: vmctx.BlockHeight(), + if params.UnlockDuration != 0 { + self.InitialBalance = vmctx.Message().Value + self.UnlockDuration = params.UnlockDuration + self.StartingBlock = vmctx.BlockHeight() } head, err := vmctx.Storage().Put(self) From 7532f92c988b9ce27fa7fef6dbec645a470847b9 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 22 Oct 2019 21:58:51 +0200 Subject: [PATCH 004/230] Cleanup GetBalance call and warn if actor does not exist License: MIT Signed-off-by: Jakub Sztandera --- chain/stmgr/stmgr.go | 2 +- cli/wallet.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ecfc51648..dd288b743 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -217,7 +217,7 @@ func (sm *StateManager) GetActor(addr address.Address, ts *types.TipSet) (*types func (sm *StateManager) GetBalance(addr address.Address, ts *types.TipSet) (types.BigInt, error) { act, err := sm.GetActor(addr, ts) if err != nil { - return types.BigInt{}, xerrors.Errorf("get actor: %w", err) + return types.NewInt(0), xerrors.Errorf("get actor: %w", err) } return act.Balance, nil diff --git a/cli/wallet.go b/cli/wallet.go index cb0cbbdd3..4af3eb0f5 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -3,6 +3,7 @@ package cli import ( "encoding/hex" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -101,7 +102,10 @@ var walletBalance = &cli.Command{ } balance, err := api.WalletBalance(ctx, addr) - if err != nil { + + if errors.Is(err, types.ErrActorNotFound) { + log.Warnf("actor not found with address: %s", addr) + } else if err != nil { return err } From d265f4a04f55927f9db1baef6a604b3701ff48d3 Mon Sep 17 00:00:00 2001 From: waynewyang Date: Wed, 23 Oct 2019 10:47:45 +0800 Subject: [PATCH 005/230] fix method:InitActorState.AddActor --- chain/actors/actor_init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index 0e259f2e2..5cd46957b 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -175,7 +175,6 @@ func IsSingletonActor(code cid.Cid) bool { func (ias *InitActorState) AddActor(cst *hamt.CborIpldStore, addr address.Address) (address.Address, error) { nid := ias.NextID - ias.NextID++ amap, err := hamt.LoadNode(context.TODO(), cst, ias.AddressMap) if err != nil { @@ -195,6 +194,7 @@ func (ias *InitActorState) AddActor(cst *hamt.CborIpldStore, addr address.Addres return address.Undef, err } ias.AddressMap = ncid + ias.NextID++ return NewIDAddress(nid) } From 7b6d67ec1e3db6da0584e66dc0076f22ee1feac8 Mon Sep 17 00:00:00 2001 From: waynewyang Date: Wed, 23 Oct 2019 11:51:48 +0800 Subject: [PATCH 006/230] Check if fromaddress equal workaddress when CommitSector --- chain/actors/actor_miner.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index de7e4d225..404051222 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -225,13 +225,17 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, err } + if vmctx.Message().From != mi.Worker { + return nil, aerrors.New(1, "not authorized to commit sector for miner") + } + // TODO: this needs to get normalized to either the ID address or the actor address maddr := vmctx.Message().To if ok, err := ValidatePoRep(maddr, mi.SectorSize, params); err != nil { return nil, err } else if !ok { - return nil, aerrors.New(1, "bad proof!") + return nil, aerrors.New(2, "bad proof!") } // make sure the miner isnt trying to submit a pre-existing sector @@ -240,7 +244,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, err } if !unique { - return nil, aerrors.New(2, "sector already committed!") + return nil, aerrors.New(3, "sector already committed!") } // Power of the miner after adding this sector @@ -248,7 +252,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex collateralRequired := CollateralForPower(futurePower) if act.Balance.LessThan(collateralRequired) { - return nil, aerrors.New(3, "not enough collateral") + return nil, aerrors.New(4, "not enough collateral") } // Note: There must exist a unique index in the miner's sector set for each From fcf928ec76acaf1840a784304193a7fcc24d850b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Oct 2019 08:16:59 +0200 Subject: [PATCH 007/230] actors: Initial structure for StorageMarketActor --- chain/actors/actor_storagemarket.go | 205 ++++++++++++++++++++++++++++ chain/actors/cbor_gen.go | 166 ++++++++++++++++++++++ gen/main.go | 3 + 3 files changed, 374 insertions(+) create mode 100644 chain/actors/actor_storagemarket.go diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go new file mode 100644 index 000000000..9b7d3ffde --- /dev/null +++ b/chain/actors/actor_storagemarket.go @@ -0,0 +1,205 @@ +package actors + +import ( + "github.com/ipfs/go-cid" + "github.com/ipfs/go-hamt-ipld" + + "github.com/filecoin-project/go-lotus/chain/actors/aerrors" + "github.com/filecoin-project/go-lotus/chain/address" + "github.com/filecoin-project/go-lotus/chain/types" +) + +type StorageMarketActor struct{} + +type smaMethods struct { + Constructor uint64 + WithdrawBalance uint64 + AddBalance uint64 + CheckLockedBalance uint64 + PublishStorageDeals uint64 + HandleCronAction uint64 + SettleExpiredDeals uint64 + ProcessStorageDealsPayment uint64 + SlashStorageDealCollateral uint64 + GetLastExpirationFromDealIDs uint64 +} + +var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + +func (sma StorageMarketActor) Exports() []interface{} { + return []interface{}{ + // 2: sma.WithdrawBalance, + // 3: sma.AddBalance, + // 4: sma.CheckLockedBalance, + // 5: sma.PublishStorageDeals, + // 6: sma.HandleCronAction, + // 7: sma.SettleExpiredDeals, + // 8: sma.ProcessStorageDealsPayment, + // 9: sma.SlashStorageDealCollateral, + // 10: sma.GetLastExpirationFromDealIDs, + } +} + +type StorageParticipantBalance struct { + Locked types.BigInt + Available types.BigInt +} + +type StorageMarketState struct { + Balances cid.Cid // hamt + Deals cid.Cid // amt +} + +type WithdrawBalanceParams struct { + Balance types.BigInt +} + +func (sma StorageMarketActor) WithdrawBalance(act *types.Actor, vmctx types.VMContext, params *WithdrawBalanceParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + b, bnd, err := sma.getBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + if err != nil { + return nil, aerrors.Wrap(err, "could not get balance") + } + + balance := b[0] + + if balance.Available.LessThan(params.Balance) { + return nil, aerrors.Newf(1, "can not withdraw more funds than available: %s > %s", params.Balance, b[0].Available) + } + + balance.Available = types.BigSub(balance.Available, params.Balance) + + _, err = vmctx.Send(vmctx.Message().From, 0, params.Balance, nil) + if err != nil { + return nil, aerrors.Wrap(err, "sending funds failed") + } + + bcid, err := sma.setBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + vmctx.Message().From: balance, + }) + if err != nil { + return nil, err + } + + self.Balances = bcid + + nroot, err := vmctx.Storage().Put(&self) + if err != nil { + return nil, err + } + + return nil, vmctx.Storage().Commit(old, nroot) +} + +func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + b, bnd, err := sma.getBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + if err != nil { + return nil, aerrors.Wrap(err, "could not get balance") + } + + balance := b[0] + + balance.Available = types.BigAdd(balance.Available, vmctx.Message().Value) + + bcid, err := sma.setBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + vmctx.Message().From: balance, + }) + if err != nil { + return nil, err + } + + self.Balances = bcid + + nroot, err := vmctx.Storage().Put(&self) + if err != nil { + return nil, err + } + + return nil, vmctx.Storage().Commit(old, nroot) +} + +func (sma StorageMarketActor) setBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { + for addr, b := range set { + if err := nd.Set(vmctx.Context(), string(addr.Bytes()), b); err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") + } + } + if err := nd.Flush(vmctx.Context()); err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "flushing balance hamt") + } + + c, err := vmctx.Ipld().Put(vmctx.Context(), nd) + if err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "failed to balances storage") + } + return c, nil +} + +func (sma StorageMarketActor) getBalances(vmctx types.VMContext, rcid cid.Cid, addrs []address.Address) ([]StorageParticipantBalance, *hamt.Node, ActorError) { + nd, err := hamt.LoadNode(vmctx.Context(), vmctx.Ipld(), rcid) + if err != nil { + return nil, nil, aerrors.HandleExternalError(err, "failed to load miner set") + } + + out := make([]StorageParticipantBalance, len(addrs)) + + for i, a := range addrs { + var balance StorageParticipantBalance + err = nd.Find(vmctx.Context(), string(a.Bytes()), &balance) + switch err { + case hamt.ErrNotFound: + out[i] = StorageParticipantBalance{ + Locked: types.NewInt(0), + Available: types.NewInt(0), + } + case nil: + out[i] = balance + default: + return nil, nil, aerrors.HandleExternalError(err, "failed to do set lookup") + } + + } + + return out, nd, nil +} + +/* +func (sma StorageMarketActor) CheckLockedBalance(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) HandleCronAction(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) SettleExpiredDeals(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) SlashStorageDealCollateral(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} + +func (sma StorageMarketActor) GetLastExpirationFromDealIDs(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { + +} +*/ diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index d39a81af2..d11b79e1c 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -2868,3 +2868,169 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { } return nil } + +func (t *StorageParticipantBalance) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Locked (types.BigInt) + if err := t.Locked.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Available (types.BigInt) + if err := t.Available.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *StorageParticipantBalance) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Locked (types.BigInt) + + { + + if err := t.Locked.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Available (types.BigInt) + + { + + if err := t.Available.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *StorageMarketState) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Balances (cid.Cid) + + if err := cbg.WriteCid(w, t.Balances); err != nil { + return xerrors.Errorf("failed to write cid field t.Balances: %w", err) + } + + // t.t.Deals (cid.Cid) + + if err := cbg.WriteCid(w, t.Deals); err != nil { + return xerrors.Errorf("failed to write cid field t.Deals: %w", err) + } + + return nil +} + +func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Balances (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Balances: %w", err) + } + + t.Balances = c + + } + // t.t.Deals (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Deals: %w", err) + } + + t.Deals = c + + } + return nil +} + +func (t *WithdrawBalanceParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Balance (types.BigInt) + if err := t.Balance.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *WithdrawBalanceParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Balance (types.BigInt) + + { + + if err := t.Balance.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} diff --git a/gen/main.go b/gen/main.go index 7dfcb0dbb..6ebeca03b 100644 --- a/gen/main.go +++ b/gen/main.go @@ -75,6 +75,9 @@ func main() { actors.ArbitrateConsensusFaultParams{}, actors.PledgeCollateralParams{}, actors.MinerSlashConsensusFault{}, + actors.StorageParticipantBalance{}, + actors.StorageMarketState{}, + actors.WithdrawBalanceParams{}, ) if err != nil { fmt.Println(err) From 8638cd25f5ba04e9e7ecd518bb749c3ca7c4e7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 19 Oct 2019 06:54:22 +0200 Subject: [PATCH 008/230] actors: implement sma.PublishStorageDeals --- chain/actors/actor_storagemarket.go | 197 ++++++++++++++- chain/actors/cbor_gen.go | 371 +++++++++++++++++++++++++++- gen/main.go | 4 + 3 files changed, 557 insertions(+), 15 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 9b7d3ffde..7ce136ff5 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -1,6 +1,9 @@ package actors import ( + "bytes" + + "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" @@ -28,10 +31,10 @@ var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} func (sma StorageMarketActor) Exports() []interface{} { return []interface{}{ - // 2: sma.WithdrawBalance, - // 3: sma.AddBalance, + 2: sma.WithdrawBalance, + 3: sma.AddBalance, // 4: sma.CheckLockedBalance, - // 5: sma.PublishStorageDeals, + 5: sma.PublishStorageDeals, // 6: sma.HandleCronAction, // 7: sma.SettleExpiredDeals, // 8: sma.ProcessStorageDealsPayment, @@ -46,8 +49,26 @@ type StorageParticipantBalance struct { } type StorageMarketState struct { - Balances cid.Cid // hamt - Deals cid.Cid // amt + Balances cid.Cid // hamt + Deals cid.Cid // amt + NextDealID uint64 // TODO: amt.LastIndex() +} + +type StorageDealProposal struct { + PieceRef cid.Cid // can this mess anything up? Should this just be cid bytes + PieceSize uint64 + Client address.Address + Provider address.Address + ProposalExpiration uint64 + DealExpiration uint64 + StoragePrice types.BigInt + StorageCollateral types.BigInt + ProposerSignature types.Signature +} + +type StorageDeal struct { + Proposal StorageDealProposal + CounterSignature types.Signature } type WithdrawBalanceParams struct { @@ -61,7 +82,7 @@ func (sma StorageMarketActor) WithdrawBalance(act *types.Actor, vmctx types.VMCo return nil, err } - b, bnd, err := sma.getBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) if err != nil { return nil, aerrors.Wrap(err, "could not get balance") } @@ -79,7 +100,7 @@ func (sma StorageMarketActor) WithdrawBalance(act *types.Actor, vmctx types.VMCo return nil, aerrors.Wrap(err, "sending funds failed") } - bcid, err := sma.setBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + bcid, err := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ vmctx.Message().From: balance, }) if err != nil { @@ -103,7 +124,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext return nil, err } - b, bnd, err := sma.getBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) if err != nil { return nil, aerrors.Wrap(err, "could not get balance") } @@ -112,7 +133,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext balance.Available = types.BigAdd(balance.Available, vmctx.Message().Value) - bcid, err := sma.setBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + bcid, err := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ vmctx.Message().From: balance, }) if err != nil { @@ -129,7 +150,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext return nil, vmctx.Storage().Commit(old, nroot) } -func (sma StorageMarketActor) setBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { +func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { for addr, b := range set { if err := nd.Set(vmctx.Context(), string(addr.Bytes()), b); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") @@ -146,7 +167,7 @@ func (sma StorageMarketActor) setBalances(vmctx types.VMContext, nd *hamt.Node, return c, nil } -func (sma StorageMarketActor) getBalances(vmctx types.VMContext, rcid cid.Cid, addrs []address.Address) ([]StorageParticipantBalance, *hamt.Node, ActorError) { +func getMarketBalances(vmctx types.VMContext, rcid cid.Cid, addrs []address.Address) ([]StorageParticipantBalance, *hamt.Node, ActorError) { nd, err := hamt.LoadNode(vmctx.Context(), vmctx.Ipld(), rcid) if err != nil { return nil, nil, aerrors.HandleExternalError(err, "failed to load miner set") @@ -178,11 +199,161 @@ func (sma StorageMarketActor) getBalances(vmctx types.VMContext, rcid cid.Cid, a func (sma StorageMarketActor) CheckLockedBalance(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { } +*/ -func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { - +type PublishStorageDealsParams struct { + Deals []StorageDeal } +type PublishStorageDealResponse struct { + DealIDs []uint64 +} + +func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.VMContext, params *PublishStorageDealsParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) + if err != nil { + // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal + return nil, aerrors.HandleExternalError(err, "loading deals amt") + } + + // todo: handle duplicate deals + + out := PublishStorageDealResponse{ + DealIDs: make([]uint64, len(params.Deals)), + } + + for i, deal := range params.Deals { + if err := self.validateDeal(vmctx, deal); err != nil { + return nil, err + } + + err := deals.Set(self.NextDealID, deal) + if err != nil { + return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT") + } + out.DealIDs[i] = self.NextDealID + + self.NextDealID++ + } + + dealsCid, err := deals.Flush() + if err != nil { + return nil, aerrors.HandleExternalError(err, "saving deals AMT") + } + + self.Deals = dealsCid + + nroot, err := vmctx.Storage().Put(&self) + if err != nil { + return nil, aerrors.HandleExternalError(err, "storing state failed") + } + + aerr := vmctx.Storage().Commit(old, nroot) + if aerr != nil { + return nil, aerr + } + + var outBuf bytes.Buffer + if err := out.MarshalCBOR(&outBuf); err != nil { + return nil, aerrors.HandleExternalError(err, "serialising output") + } + + return outBuf.Bytes(), nil +} + +func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal) aerrors.ActorError { + // REVIEW: just > ? + if vmctx.BlockHeight() >= deal.Proposal.ProposalExpiration { + return aerrors.New(1, "deal proposal already expired") + } + if vmctx.BlockHeight() >= deal.Proposal.DealExpiration { + return aerrors.New(2, "deal proposal already expired") + } + + var proposalBuf bytes.Buffer + err := deal.Proposal.MarshalCBOR(&proposalBuf) + if err != nil { + return aerrors.HandleExternalError(err, "serializing deal proposal failed") + } + + err = deal.Proposal.ProposerSignature.Verify(deal.Proposal.Client, proposalBuf.Bytes()) + if err != nil { + return aerrors.HandleExternalError(err, "verifying proposer signature") + } + + var dealBuf bytes.Buffer + err = deal.MarshalCBOR(&dealBuf) + if err != nil { + return aerrors.HandleExternalError(err, "serializing deal failed") + } + + err = deal.CounterSignature.Verify(deal.Proposal.Provider, dealBuf.Bytes()) + if err != nil { + return aerrors.HandleExternalError(err, "verifying provider signature") + } + + // TODO: maybe this is actually fine + if vmctx.Message().From != deal.Proposal.Provider && vmctx.Message().From != deal.Proposal.Client { + return aerrors.New(3, "message not sent by deal participant") + } + + // TODO: REVIEW: Do we want to check if provider exists in the power actor? + + // TODO: do some caching (changes gas so needs to be in spec too) + b, bnd, aerr := getMarketBalances(vmctx, self.Balances, []address.Address{ + deal.Proposal.Client, + deal.Proposal.Provider, + }) + if aerr != nil { + return aerrors.Wrap(aerr, "getting client, and provider balances") + } + clientBalance := b[0] + providerBalance := b[1] + + if clientBalance.Available.LessThan(deal.Proposal.StoragePrice) { + return aerrors.Newf(4, "client doesn't have enough available funds to cover StoragePrice; %d < %d", clientBalance.Available, deal.Proposal.StoragePrice) + } + + clientBalance = lockFunds(clientBalance, deal.Proposal.StoragePrice) + + // TODO: REVIEW: Not clear who pays for this + if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) { + return aerrors.Newf(5, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral) + } + + providerBalance = lockFunds(providerBalance, deal.Proposal.StorageCollateral) + + // TODO: piece checks (e.g. size > sectorSize)? + + bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + deal.Proposal.Client: clientBalance, + deal.Proposal.Provider: providerBalance, + }) + if aerr != nil { + return aerr + } + + self.Balances = bcid + + return nil +} + +func lockFunds(p StorageParticipantBalance, amt types.BigInt) StorageParticipantBalance { + p.Available, p.Locked = transferFunds(p.Available, p.Locked, amt) + return p +} + +func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) { + return types.BigSub(from, amt), types.BigAdd(to, amt) +} + +/* func (sma StorageMarketActor) HandleCronAction(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index d11b79e1c..a4db730ca 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -2931,7 +2931,7 @@ func (t *StorageMarketState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{130}); err != nil { + if _, err := w.Write([]byte{131}); err != nil { return err } @@ -2947,6 +2947,10 @@ func (t *StorageMarketState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Deals: %w", err) } + // t.t.NextDealID (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextDealID)); err != nil { + return err + } return nil } @@ -2961,7 +2965,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 2 { + if extra != 3 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -2989,6 +2993,16 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { t.Deals = c } + // t.t.NextDealID (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NextDealID = extra return nil } @@ -3034,3 +3048,356 @@ func (t *WithdrawBalanceParams) UnmarshalCBOR(r io.Reader) error { } return nil } + +func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{137}); err != nil { + return err + } + + // t.t.PieceRef (cid.Cid) + + if err := cbg.WriteCid(w, t.PieceRef); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceRef: %w", err) + } + + // t.t.PieceSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSize)); err != nil { + return err + } + + // t.t.Client (address.Address) + if err := t.Client.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Provider (address.Address) + if err := t.Provider.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ProposalExpiration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + return err + } + + // t.t.DealExpiration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DealExpiration)); err != nil { + return err + } + + // t.t.StoragePrice (types.BigInt) + if err := t.StoragePrice.MarshalCBOR(w); err != nil { + return err + } + + // t.t.StorageCollateral (types.BigInt) + if err := t.StorageCollateral.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ProposerSignature (types.Signature) + if err := t.ProposerSignature.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 9 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.PieceRef (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceRef: %w", err) + } + + t.PieceRef = c + + } + // t.t.PieceSize (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PieceSize = extra + // t.t.Client (address.Address) + + { + + if err := t.Client.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Provider (address.Address) + + { + + if err := t.Provider.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ProposalExpiration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ProposalExpiration = extra + // t.t.DealExpiration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealExpiration = extra + // t.t.StoragePrice (types.BigInt) + + { + + if err := t.StoragePrice.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.StorageCollateral (types.BigInt) + + { + + if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ProposerSignature (types.Signature) + + { + + if err := t.ProposerSignature.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *StorageDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Proposal (actors.StorageDealProposal) + if err := t.Proposal.MarshalCBOR(w); err != nil { + return err + } + + // t.t.CounterSignature (types.Signature) + if err := t.CounterSignature.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Proposal (actors.StorageDealProposal) + + { + + if err := t.Proposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.CounterSignature (types.Signature) + + { + + if err := t.CounterSignature.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Deals ([]actors.StorageDeal) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { + return err + } + for _, v := range t.Deals { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Deals ([]actors.StorageDeal) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Deals: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Deals = make([]StorageDeal, extra) + } + for i := 0; i < int(extra); i++ { + + var v StorageDeal + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Deals[i] = v + } + + return nil +} + +func (t *PublishStorageDealResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *PublishStorageDealResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + return nil +} diff --git a/gen/main.go b/gen/main.go index 6ebeca03b..17a4a38b5 100644 --- a/gen/main.go +++ b/gen/main.go @@ -78,6 +78,10 @@ func main() { actors.StorageParticipantBalance{}, actors.StorageMarketState{}, actors.WithdrawBalanceParams{}, + actors.StorageDealProposal{}, + actors.StorageDeal{}, + actors.PublishStorageDealsParams{}, + actors.PublishStorageDealResponse{}, ) if err != nil { fmt.Println(err) From c41dd4efeb044f1b9ac9d830a24c7ef49e208d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 19 Oct 2019 10:06:41 +0200 Subject: [PATCH 009/230] actora: register storagemarket --- chain/actors/actor_init.go | 2 +- chain/actors/actor_miner.go | 4 ++-- chain/actors/actor_storagepower_test.go | 16 ++++++++-------- chain/actors/actors.go | 17 ++++++++++------- chain/actors/actors_test.go | 2 +- chain/gen/utils.go | 8 ++++---- chain/stmgr/utils.go | 6 +++--- chain/store/weight.go | 4 ++-- chain/sync.go | 2 +- chain/vm/invoker.go | 3 ++- cli/createminer.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- node/impl/full/state.go | 6 +++--- 13 files changed, 39 insertions(+), 35 deletions(-) diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index 64a154b71..fdff41589 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -170,7 +170,7 @@ func IsBuiltinActor(code cid.Cid) bool { } func IsSingletonActor(code cid.Cid) bool { - return code == StorageMarketActorCodeCid || code == InitActorCodeCid + return code == StoragePowerActorCodeCid || code == InitActorCodeCid } func (ias *InitActorState) AddActor(cst *hamt.CborIpldStore, addr address.Address) (address.Address, error) { diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index de7e4d225..ba805cd2d 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -433,7 +433,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext, return nil, err } - _, err = vmctx.Send(StorageMarketAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc) + _, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc) if err != nil { return nil, err } @@ -753,7 +753,7 @@ type MinerSlashConsensusFault struct { } func (sma StorageMinerActor) SlashConsensusFault(act *types.Actor, vmctx types.VMContext, params *MinerSlashConsensusFault) ([]byte, ActorError) { - if vmctx.Message().From != StorageMarketAddress { + if vmctx.Message().From != StoragePowerAddress { return nil, aerrors.New(1, "SlashConsensusFault may only be called by the storage market actor") } diff --git a/chain/actors/actor_storagepower_test.go b/chain/actors/actor_storagepower_test.go index cc4ef921c..76af74c9d 100644 --- a/chain/actors/actor_storagepower_test.go +++ b/chain/actors/actor_storagepower_test.go @@ -37,7 +37,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { // cheating the bootstrapping problem cheatStorageMarketTotal(t, h.vm, h.cs.Blockstore()) - ret, _ := h.InvokeWithValue(t, ownerAddr, StorageMarketAddress, SPAMethods.CreateStorageMiner, + ret, _ := h.InvokeWithValue(t, ownerAddr, StoragePowerAddress, SPAMethods.CreateStorageMiner, types.NewInt(500000), &CreateStorageMinerParams{ Owner: ownerAddr, @@ -52,7 +52,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { } { - ret, _ := h.Invoke(t, ownerAddr, StorageMarketAddress, SPAMethods.IsMiner, + ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.IsMiner, &IsMinerParam{Addr: minerAddr}) ApplyOK(t, ret) @@ -68,7 +68,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { } { - ret, _ := h.Invoke(t, ownerAddr, StorageMarketAddress, SPAMethods.PowerLookup, + ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.PowerLookup, &PowerLookupParams{Miner: minerAddr}) ApplyOK(t, ret) power := types.BigFromBytes(ret.Return) @@ -93,7 +93,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { signBlock(t, h.w, workerAddr, b1) signBlock(t, h.w, workerAddr, b2) - ret, _ := h.Invoke(t, ownerAddr, StorageMarketAddress, SPAMethods.ArbitrateConsensusFault, + ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.ArbitrateConsensusFault, &ArbitrateConsensusFaultParams{ Block1: b1, Block2: b2, @@ -102,13 +102,13 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { } { - ret, _ := h.Invoke(t, ownerAddr, StorageMarketAddress, SPAMethods.PowerLookup, + ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.PowerLookup, &PowerLookupParams{Miner: minerAddr}) assert.Equal(t, ret.ExitCode, byte(1)) } { - ret, _ := h.Invoke(t, ownerAddr, StorageMarketAddress, SPAMethods.IsMiner, &IsMinerParam{minerAddr}) + ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.IsMiner, &IsMinerParam{minerAddr}) ApplyOK(t, ret) assert.Equal(t, ret.Return, cbg.CborBoolFalse) } @@ -117,7 +117,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { func cheatStorageMarketTotal(t *testing.T, vm *vm.VM, bs bstore.Blockstore) { t.Helper() - sma, err := vm.StateTree().GetActor(StorageMarketAddress) + sma, err := vm.StateTree().GetActor(StoragePowerAddress) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func cheatStorageMarketTotal(t *testing.T, vm *vm.VM, bs bstore.Blockstore) { sma.Head = c - if err := vm.StateTree().SetActor(StorageMarketAddress, sma); err != nil { + if err := vm.StateTree().SetActor(StoragePowerAddress, sma); err != nil { t.Fatal(err) } } diff --git a/chain/actors/actors.go b/chain/actors/actors.go index 72d3b96d5..0f18964cb 100644 --- a/chain/actors/actors.go +++ b/chain/actors/actors.go @@ -8,6 +8,7 @@ import ( ) var AccountActorCodeCid cid.Cid +var StoragePowerActorCodeCid cid.Cid var StorageMarketActorCodeCid cid.Cid var StorageMinerCodeCid cid.Cid var MultisigActorCodeCid cid.Cid @@ -16,7 +17,8 @@ var PaymentChannelActorCodeCid cid.Cid var InitActorAddress = mustIDAddress(0) var NetworkAddress = mustIDAddress(1) -var StorageMarketAddress = mustIDAddress(2) +var StoragePowerAddress = mustIDAddress(2) +var StorageMarketAddress = mustIDAddress(3) // TODO: missing from spec var BurntFundsAddress = mustIDAddress(99) func mustIDAddress(i uint64) address.Address { @@ -37,10 +39,11 @@ func init() { return c } - AccountActorCodeCid = mustSum("account") - StorageMarketActorCodeCid = mustSum("smarket") - StorageMinerCodeCid = mustSum("sminer") - MultisigActorCodeCid = mustSum("multisig") - InitActorCodeCid = mustSum("init") - PaymentChannelActorCodeCid = mustSum("paych") + AccountActorCodeCid = mustSum("filecoin/1.0/AccountActor") + StoragePowerActorCodeCid = mustSum("filecoin/1.0/StoragePowerActor") + StorageMarketActorCodeCid = mustSum("filecoin/1.0/StorageMarketActor") + StorageMinerCodeCid = mustSum("filecoin/1.0/StorageMinerActor") + MultisigActorCodeCid = mustSum("filecoin/1.0/MultisigActor") + InitActorCodeCid = mustSum("filecoin/1.0/InitActor") + PaymentChannelActorCodeCid = mustSum("filecoin/1.0/PaymentChannelActor") } diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 36902de75..27946664c 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -128,7 +128,7 @@ func TestStorageMarketActorCreateMiner(t *testing.T) { } msg := &types.Message{ - To: StorageMarketAddress, + To: StoragePowerAddress, From: from, Method: SPAMethods.CreateStorageMiner, Params: enc, diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 6a1a5ca34..06ade0030 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -94,7 +94,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types return nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(actors.StorageMarketAddress, smact); err != nil { + if err := state.SetActor(actors.StoragePowerAddress, smact); err != nil { return nil, xerrors.Errorf("set storage market actor: %w", err) } @@ -154,7 +154,7 @@ func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { } return &types.Actor{ - Code: actors.StorageMarketActorCodeCid, + Code: actors.StoragePowerActorCodeCid, Head: stcid, Nonce: 0, Balance: types.NewInt(0), @@ -202,7 +202,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid // TODO: hardcoding 7000000 here is a little fragile, it changes any // time anyone changes the initial account allocations - rval, err := doExecValue(ctx, vm, actors.StorageMarketAddress, owner, types.FromFil(6500), actors.SPAMethods.CreateStorageMiner, params) + rval, err := doExecValue(ctx, vm, actors.StoragePowerAddress, owner, types.FromFil(6500), actors.SPAMethods.CreateStorageMiner, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err) } @@ -216,7 +216,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid params = mustEnc(&actors.UpdateStorageParams{Delta: types.NewInt(5000)}) - _, err = doExec(ctx, vm, actors.StorageMarketAddress, maddr, actors.SPAMethods.UpdateStorage, params) + _, err = doExec(ctx, vm, actors.StoragePowerAddress, maddr, actors.SPAMethods.UpdateStorage, params) if err != nil { return cid.Undef, xerrors.Errorf("failed to update total storage: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 025a8deb7..8a7398b9a 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -80,7 +80,7 @@ func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr add } ret, err := sm.Call(ctx, &types.Message{ From: maddr, - To: actors.StorageMarketAddress, + To: actors.StoragePowerAddress, Method: actors.SPAMethods.PowerLookup, Params: enc, }, ts) @@ -95,8 +95,8 @@ func GetPower(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr add } ret, err := sm.Call(ctx, &types.Message{ - From: actors.StorageMarketAddress, - To: actors.StorageMarketAddress, + From: actors.StoragePowerAddress, + To: actors.StoragePowerAddress, Method: actors.SPAMethods.GetTotalStorage, }, ts) if err != nil { diff --git a/chain/store/weight.go b/chain/store/weight.go index cf992554a..7c1f80ceb 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -23,8 +23,8 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn // >>> wFunction(totalPowerAtTipset(ts)) * 2^8 <<< + (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den) ret, err := cs.call(ctx, &types.Message{ - From: actors.StorageMarketAddress, - To: actors.StorageMarketAddress, + From: actors.StoragePowerAddress, + To: actors.StoragePowerAddress, Method: actors.SPAMethods.GetTotalStorage, }, ts) if err != nil { diff --git a/chain/sync.go b/chain/sync.go index def95c25e..c27532e09 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -398,7 +398,7 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b } ret, err := syncer.sm.Call(ctx, &types.Message{ - To: actors.StorageMarketAddress, + To: actors.StoragePowerAddress, From: maddr, Method: actors.SPAMethods.IsMiner, Params: enc, diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index 713f64d39..c91109734 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -30,7 +30,8 @@ func newInvoker() *invoker { // add builtInCode using: register(cid, singleton) inv.register(actors.InitActorCodeCid, actors.InitActor{}, actors.InitActorState{}) - inv.register(actors.StorageMarketActorCodeCid, actors.StoragePowerActor{}, actors.StoragePowerState{}) + inv.register(actors.StoragePowerActorCodeCid, actors.StoragePowerActor{}, actors.StoragePowerState{}) + inv.register(actors.StorageMarketActorCodeCid, actors.StorageMarketActor{}, actors.StorageMarketState{}) inv.register(actors.StorageMinerCodeCid, actors.StorageMinerActor{}, actors.StorageMinerActorState{}) inv.register(actors.MultisigActorCodeCid, actors.MultiSigActor{}, actors.MultiSigActorState{}) inv.register(actors.PaymentChannelActorCodeCid, actors.PaymentChannelActor{}, actors.PaymentChannelActorState{}) diff --git a/cli/createminer.go b/cli/createminer.go index 551528613..25640b5fa 100644 --- a/cli/createminer.go +++ b/cli/createminer.go @@ -69,7 +69,7 @@ var createMinerCmd = &cli.Command{ } msg := &types.Message{ - To: actors.StorageMarketAddress, + To: actors.StoragePowerAddress, From: addr, Method: actors.SPAMethods.CreateStorageMiner, Params: params, diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a0085fe81..d4e7a7235 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -303,7 +303,7 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, c } createStorageMinerMsg := &types.Message{ - To: actors.StorageMarketAddress, + To: actors.StoragePowerAddress, From: owner, Value: collateral, diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 135317ca9..e357c0814 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -90,8 +90,8 @@ func (a *StateAPI) StatePledgeCollateral(ctx context.Context, ts *types.TipSet) } ret, aerr := a.StateManager.Call(ctx, &types.Message{ - From: actors.StorageMarketAddress, - To: actors.StorageMarketAddress, + From: actors.StoragePowerAddress, + To: actors.StoragePowerAddress, Method: actors.SPAMethods.PledgeCollateralForSize, Params: param, @@ -210,7 +210,7 @@ func (a *StateAPI) StateWaitMsg(ctx context.Context, msg cid.Cid) (*api.MsgWait, func (a *StateAPI) StateListMiners(ctx context.Context, ts *types.TipSet) ([]address.Address, error) { var state actors.StoragePowerState - if _, err := a.StateManager.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { + if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { return nil, err } From 6bade2eb23f44282bc7afbb44c4a39ac76e58bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 19 Oct 2019 12:20:33 +0200 Subject: [PATCH 010/230] on chain deals: More actor methods --- chain/actors/actor_miner.go | 8 ++ chain/actors/actor_storagemarket.go | 181 ++++++++++++++++++++++++++-- chain/actors/cbor_gen.go | 138 +++++++++++++++++++++ gen/main.go | 2 + 4 files changed, 321 insertions(+), 8 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index ba805cd2d..450a4c905 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -213,6 +213,14 @@ type CommitSectorParams struct { Proof []byte } +type OnChainSealVerifyInfo struct { + SealedCID cid.Cid // CommR .. TODO: spec says cid, but it feela weird + Epoch uint64 + Proof []byte + DealIDs []uint64 + SectorNumber uint64 +} + func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *CommitSectorParams) ([]byte, ActorError) { ctx := context.TODO() oldstate, self, err := loadState(vmctx) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 7ce136ff5..7f904048d 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -6,7 +6,9 @@ import ( "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" + cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/go-lotus/build" "github.com/filecoin-project/go-lotus/chain/actors/aerrors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/types" @@ -25,9 +27,10 @@ type smaMethods struct { ProcessStorageDealsPayment uint64 SlashStorageDealCollateral uint64 GetLastExpirationFromDealIDs uint64 + ActivateStorageDeals uint64 } -var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} +var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} func (sma StorageMarketActor) Exports() []interface{} { return []interface{}{ @@ -40,6 +43,7 @@ func (sma StorageMarketActor) Exports() []interface{} { // 8: sma.ProcessStorageDealsPayment, // 9: sma.SlashStorageDealCollateral, // 10: sma.GetLastExpirationFromDealIDs, + 11: sma.ActivateStorageDeals, // TODO: move under PublishStorageDeals after specs team approves } } @@ -49,9 +53,10 @@ type StorageParticipantBalance struct { } type StorageMarketState struct { - Balances cid.Cid // hamt - Deals cid.Cid // amt - NextDealID uint64 // TODO: amt.LastIndex() + Balances cid.Cid // hamt + Deals cid.Cid // amt + + NextDealID uint64 // TODO: amt.LastIndex() } type StorageDealProposal struct { @@ -71,6 +76,11 @@ type StorageDeal struct { CounterSignature types.Signature } +type OnChainDeal struct { + Deal StorageDeal + ActivationEpoch uint64 // 0 = inactive +} + type WithdrawBalanceParams struct { Balance types.BigInt } @@ -209,6 +219,9 @@ type PublishStorageDealResponse struct { DealIDs []uint64 } +// TODO: spec says 'call by CommitSector in StorageMiningSubsystem', and then +// says that this should be called before CommitSector. For now assuming that +// they meant 2 separate methods, See 'ActivateStorageDeals' below func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.VMContext, params *PublishStorageDealsParams) ([]byte, ActorError) { var self StorageMarketState old := vmctx.Storage().GetHead() @@ -233,7 +246,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return nil, err } - err := deals.Set(self.NextDealID, deal) + err := deals.Set(self.NextDealID, OnChainDeal{Deal: deal}) if err != nil { return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT") } @@ -275,6 +288,9 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage if vmctx.BlockHeight() >= deal.Proposal.DealExpiration { return aerrors.New(2, "deal proposal already expired") } + if deal.Proposal.ProposalExpiration > deal.Proposal.DealExpiration { + return aerrors.New(3, "ProposalExpiration > DealExpiration") + } var proposalBuf bytes.Buffer err := deal.Proposal.MarshalCBOR(&proposalBuf) @@ -300,7 +316,7 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage // TODO: maybe this is actually fine if vmctx.Message().From != deal.Proposal.Provider && vmctx.Message().From != deal.Proposal.Client { - return aerrors.New(3, "message not sent by deal participant") + return aerrors.New(4, "message not sent by deal participant") } // TODO: REVIEW: Do we want to check if provider exists in the power actor? @@ -317,14 +333,14 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage providerBalance := b[1] if clientBalance.Available.LessThan(deal.Proposal.StoragePrice) { - return aerrors.Newf(4, "client doesn't have enough available funds to cover StoragePrice; %d < %d", clientBalance.Available, deal.Proposal.StoragePrice) + return aerrors.Newf(5, "client doesn't have enough available funds to cover StoragePrice; %d < %d", clientBalance.Available, deal.Proposal.StoragePrice) } clientBalance = lockFunds(clientBalance, deal.Proposal.StoragePrice) // TODO: REVIEW: Not clear who pays for this if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - return aerrors.Newf(5, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral) + return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral) } providerBalance = lockFunds(providerBalance, deal.Proposal.StorageCollateral) @@ -344,12 +360,161 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage return nil } +type ActivateStorageDealsParams struct { + Deals []uint64 +} + +func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types.VMContext, params *ActivateStorageDealsParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) + if err != nil { + // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal + return nil, aerrors.HandleExternalError(err, "loading deals amt") + } + + for _, deal := range params.Deals { + var dealInfo OnChainDeal + if err := deals.Get(deal, &dealInfo); err != nil { + return nil, aerrors.HandleExternalError(err, "getting del info failed") + } + + if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { + return nil, aerrors.New(1, "ActivateStorageDeals can only be called by deal provider") + } + + if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { + return nil, aerrors.New(2, "deal cannot be activated: proposal expired") + } + + if dealInfo.ActivationEpoch > 0 { + // this probably can't happen in practice + return nil, aerrors.New(3, "deal already active") + } + + dealInfo.ActivationEpoch = vmctx.BlockHeight() + + if err := deals.Set(deal, dealInfo); err != nil { + return nil, aerrors.HandleExternalError(err, "setting deal info in AMT failed") + } + } + + dealsCid, err := deals.Flush() + if err != nil { + return nil, aerrors.HandleExternalError(err, "saving deals AMT") + } + + self.Deals = dealsCid + + nroot, err := vmctx.Storage().Put(&self) + if err != nil { + return nil, aerrors.HandleExternalError(err, "storing state failed") + } + + aerr := vmctx.Storage().Commit(old, nroot) + if aerr != nil { + return nil, aerr + } + + return nil, nil +} + +type ProcessStorageDealsPaymentParams struct { + DealIDs []uint64 +} + +func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx types.VMContext, params *ProcessStorageDealsPaymentParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) + if err != nil { + // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal + return nil, aerrors.HandleExternalError(err, "loading deals amt") + } + + for _, deal := range params.DealIDs { + var dealInfo OnChainDeal + if err := deals.Get(deal, &dealInfo); err != nil { + return nil, aerrors.HandleExternalError(err, "getting del info failed") + } + + encoded, err := CreateExecParams(StorageMinerCodeCid, &IsMinerParam{ + Addr: vmctx.Message().From, + }) + if err != nil { + return nil, err + } + + ret, err := vmctx.Send(StoragePowerAddress, SPAMethods.IsMiner, types.NewInt(0), encoded) + if err != nil { + return nil, err + } + + if bytes.Equal(ret, cbg.CborBoolTrue) { + return nil, aerrors.New(1, "ProcessStorageDealsPayment can only be called by storage miner actors") + } + + if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { + // TODO: ??? + return nil, nil + } + + // TODO: clients probably want this to be fixed + dealDuration := dealInfo.Deal.Proposal.DealExpiration - dealInfo.ActivationEpoch + + // todo: check math (written on a plane, also tired) + // TODO: division is hard, this more than likely has some off-by-one issue + toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealDuration)) + + b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{ + dealInfo.Deal.Proposal.Client, + dealInfo.Deal.Proposal.Provider, + }) + clientBal := b[0] + providerBal := b[1] + + clientBal.Locked, providerBal.Available = transferFunds(clientBal.Locked, providerBal.Available, toPay) + + // TODO: call set once + bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ + dealInfo.Deal.Proposal.Client: clientBal, + dealInfo.Deal.Proposal.Provider: providerBal, + }) + if aerr != nil { + return nil, aerr + } + + self.Balances = bcid + } + + nroot, err := vmctx.Storage().Put(&self) + if err != nil { + return nil, aerrors.HandleExternalError(err, "storing state failed") + } + + aerr := vmctx.Storage().Commit(old, nroot) + if aerr != nil { + return nil, aerr + } + + return nil, nil +} + func lockFunds(p StorageParticipantBalance, amt types.BigInt) StorageParticipantBalance { p.Available, p.Locked = transferFunds(p.Available, p.Locked, amt) return p } func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) { + // TODO: some asserts return types.BigSub(from, amt), types.BigAdd(to, amt) } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index a4db730ca..de6ae4db3 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3401,3 +3401,141 @@ func (t *PublishStorageDealResponse) UnmarshalCBOR(r io.Reader) error { return nil } + +func (t *ActivateStorageDealsParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Deals ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { + return err + } + for _, v := range t.Deals { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *ActivateStorageDealsParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Deals ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Deals: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Deals = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.Deals slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.Deals was not a uint, instead got %d", maj) + } + + t.Deals[i] = val + } + + return nil +} + +func (t *ProcessStorageDealsPaymentParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *ProcessStorageDealsPaymentParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + return nil +} diff --git a/gen/main.go b/gen/main.go index 17a4a38b5..dd5842ed0 100644 --- a/gen/main.go +++ b/gen/main.go @@ -82,6 +82,8 @@ func main() { actors.StorageDeal{}, actors.PublishStorageDealsParams{}, actors.PublishStorageDealResponse{}, + actors.ActivateStorageDealsParams{}, + actors.ProcessStorageDealsPaymentParams{}, ) if err != nil { fmt.Println(err) From 7420dd668e133d422bd1eebf1e29f332acbd8168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Oct 2019 10:53:41 +0200 Subject: [PATCH 011/230] on chain deals: use cid bytes for PieceRef --- chain/actors/actor_storagemarket.go | 2 +- chain/actors/cbor_gen.go | 33 +++++++++++++++++------------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 7f904048d..6004c6113 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -60,7 +60,7 @@ type StorageMarketState struct { } type StorageDealProposal struct { - PieceRef cid.Cid // can this mess anything up? Should this just be cid bytes + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea PieceSize uint64 Client address.Address Provider address.Address diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index de6ae4db3..85a199270 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3058,10 +3058,12 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceRef (cid.Cid) - - if err := cbg.WriteCid(w, t.PieceRef); err != nil { - return xerrors.Errorf("failed to write cid field t.PieceRef: %w", err) + // t.t.PieceRef ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { + return err + } + if _, err := w.Write(t.PieceRef); err != nil { + return err } // t.t.PieceSize (uint64) @@ -3121,17 +3123,22 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PieceRef (cid.Cid) + // t.t.PieceRef ([]uint8) - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PieceRef: %w", err) - } - - t.PieceRef = c + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.PieceRef: array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PieceRef = make([]byte, extra) + if _, err := io.ReadFull(br, t.PieceRef); err != nil { + return err } // t.t.PieceSize (uint64) From 46a0333c9c48e495a45a0a66302ce66fceea6f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Oct 2019 20:12:11 +0200 Subject: [PATCH 012/230] on chain deals: Wip porting deal systems to storagemarket --- api/api.go | 3 + api/types.go | 11 +- api/utils.go | 27 ++ chain/actors/actor_init.go | 4 +- chain/actors/actor_miner.go | 124 ++----- chain/actors/actor_multisig_test.go | 4 +- chain/actors/actor_paych_test.go | 4 +- chain/actors/actor_storagemarket.go | 77 ++++- chain/actors/actor_storagepower.go | 2 +- chain/actors/actors.go | 26 +- chain/actors/actors_test.go | 2 +- chain/actors/harness2_test.go | 2 +- chain/deals/client.go | 83 +++-- chain/deals/client_states.go | 37 ++- chain/deals/client_utils.go | 59 ---- chain/deals/handler_states.go | 313 ------------------ chain/deals/{handler.go => provider.go} | 70 ++-- chain/deals/{asks.go => provider_asks.go} | 64 ++-- chain/deals/provider_states.go | 245 ++++++++++++++ .../{handler_utils.go => provider_utils.go} | 38 ++- chain/deals/types.go | 56 +--- chain/gen/utils.go | 14 +- chain/state/statetree.go | 4 +- chain/state/statetree_test.go | 8 +- chain/vm/invoker.go | 10 +- chain/vm/mkactor.go | 4 +- chain/vm/vm.go | 4 +- node/builder.go | 2 +- node/impl/client/client.go | 6 +- node/modules/storageminer.go | 4 +- paych/simple.go | 4 +- 31 files changed, 583 insertions(+), 728 deletions(-) create mode 100644 api/utils.go delete mode 100644 chain/deals/handler_states.go rename chain/deals/{handler.go => provider.go} (71%) rename chain/deals/{asks.go => provider_asks.go} (66%) create mode 100644 chain/deals/provider_states.go rename chain/deals/{handler_utils.go => provider_utils.go} (66%) diff --git a/api/api.go b/api/api.go index 5e6373fac..12e96d69d 100644 --- a/api/api.go +++ b/api/api.go @@ -3,6 +3,7 @@ package api import ( "context" "fmt" + "github.com/filecoin-project/lotus/chain/actors" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" @@ -11,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" @@ -132,6 +134,7 @@ type FullNode interface { StateWaitMsg(context.Context, cid.Cid) (*MsgWait, error) StateListMiners(context.Context, *types.TipSet) ([]address.Address, error) StateListActors(context.Context, *types.TipSet) ([]address.Address, error) + StateMarketBalance(context.Context, address.Address) (actors.StorageParticipantBalance, error) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) diff --git a/api/types.go b/api/types.go index 588227de2..f584345f3 100644 --- a/api/types.go +++ b/api/types.go @@ -10,12 +10,13 @@ type DealState int const ( DealUnknown = DealState(iota) - DealRejected - DealAccepted - DealStarted + DealRejected // Provider didn't like the proposal + DealAccepted // Proposal accepted, data moved + DealStaged // Data put into the sector + DealSealing // Data in process of being sealed + DealFailed - DealStaged - DealSealing + DealComplete // Internal diff --git a/api/utils.go b/api/utils.go new file mode 100644 index 000000000..5bbac426e --- /dev/null +++ b/api/utils.go @@ -0,0 +1,27 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" +) + +type SignFunc = func(context.Context, []byte) (*types.Signature, error) + +type Signer func(context.Context, address.Address, []byte) (*types.Signature, error) + +type Signable interface { + Sign(context.Context, SignFunc) error +} + +func SignWith(ctx context.Context, signer Signer, addr address.Address, signable ...Signable) error { + for _, s := range signable { + err := s.Sign(ctx, func(ctx context.Context, b []byte) (*types.Signature, error) { + return signer(ctx, addr, b) + }) + if err != nil { + return err + } + } +} diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index fdff41589..3ab1243a8 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -162,7 +162,7 @@ func (ia InitActor) Exec(act *types.Actor, vmctx types.VMContext, p *ExecParams) func IsBuiltinActor(code cid.Cid) bool { switch code { - case StorageMarketActorCodeCid, StorageMinerCodeCid, AccountActorCodeCid, InitActorCodeCid, MultisigActorCodeCid, PaymentChannelActorCodeCid: + case StorageMarketCodeCid, StoragePowerCodeCid, StorageMinerCodeCid, AccountCodeCid, InitCodeCid, MultisigCodeCid, PaymentChannelCodeCid: return true default: return false @@ -170,7 +170,7 @@ func IsBuiltinActor(code cid.Cid) bool { } func IsSingletonActor(code cid.Cid) bool { - return code == StoragePowerActorCodeCid || code == InitActorCodeCid + return code == StoragePowerCodeCid || code == InitCodeCid } func (ias *InitActorState) AddActor(cst *hamt.CborIpldStore, addr address.Address) (address.Address, error) { diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 450a4c905..7f87a2aea 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -100,29 +100,27 @@ type StorageMinerConstructorParams struct { } type maMethods struct { - Constructor uint64 - CommitSector uint64 - SubmitPoSt uint64 - SlashStorageFault uint64 - GetCurrentProvingSet uint64 - ArbitrateDeal uint64 - DePledge uint64 - GetOwner uint64 - GetWorkerAddr uint64 - GetPower uint64 - GetPeerID uint64 - GetSectorSize uint64 - UpdatePeerID uint64 - ChangeWorker uint64 - IsSlashed uint64 - IsLate uint64 - PaymentVerifyInclusion uint64 - PaymentVerifySector uint64 - AddFaults uint64 - SlashConsensusFault uint64 + Constructor uint64 + CommitSector uint64 + SubmitPoSt uint64 + SlashStorageFault uint64 + GetCurrentProvingSet uint64 + ArbitrateDeal uint64 + DePledge uint64 + GetOwner uint64 + GetWorkerAddr uint64 + GetPower uint64 + GetPeerID uint64 + GetSectorSize uint64 + UpdatePeerID uint64 + ChangeWorker uint64 + IsSlashed uint64 + IsLate uint64 + AddFaults uint64 + SlashConsensusFault uint64 } -var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20} +var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18} func (sma StorageMinerActor) Exports() []interface{} { return []interface{}{ @@ -142,10 +140,8 @@ func (sma StorageMinerActor) Exports() []interface{} { //14: sma.ChangeWorker, //15: sma.IsSlashed, //16: sma.IsLate, - 17: sma.PaymentVerifyInclusion, - 18: sma.PaymentVerifySector, - 19: sma.AddFaults, - 20: sma.SlashConsensusFault, + 17: sma.AddFaults, + 18: sma.SlashConsensusFault, } } @@ -642,84 +638,6 @@ type PaymentVerifyParams struct { Proof []byte } -type PieceInclVoucherData struct { // TODO: Update spec at https://github.com/filecoin-project/specs/blob/master/actors.md#paymentverify - CommP []byte - PieceSize types.BigInt -} - -type InclusionProof struct { - Sector uint64 // for CommD, also verifies the sector is in sector set - Proof []byte -} - -func (sma StorageMinerActor) PaymentVerifyInclusion(act *types.Actor, vmctx types.VMContext, params *PaymentVerifyParams) ([]byte, ActorError) { - // params.Extra - PieceInclVoucherData - // params.Proof - InclusionProof - - _, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } - mi, aerr := loadMinerInfo(vmctx, self) - if aerr != nil { - return nil, aerr - } - - var voucherData PieceInclVoucherData - if err := cbor.DecodeInto(params.Extra, &voucherData); err != nil { - return nil, aerrors.Absorb(err, 2, "failed to decode storage voucher data for verification") - } - var proof InclusionProof - if err := cbor.DecodeInto(params.Proof, &proof); err != nil { - return nil, aerrors.Absorb(err, 3, "failed to decode storage payment proof") - } - - ok, _, commD, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, proof.Sector) - if aerr != nil { - return nil, aerr - } - if !ok { - return nil, aerrors.New(4, "miner does not have required sector") - } - - ok, err := sectorbuilder.VerifyPieceInclusionProof(mi.SectorSize.Uint64(), voucherData.PieceSize.Uint64(), voucherData.CommP, commD, proof.Proof) - if err != nil { - return nil, aerrors.Absorb(err, 5, "verify piece inclusion proof failed") - } - if !ok { - return nil, aerrors.New(6, "piece inclusion proof was invalid") - } - - return nil, nil -} - -func (sma StorageMinerActor) PaymentVerifySector(act *types.Actor, vmctx types.VMContext, params *PaymentVerifyParams) ([]byte, ActorError) { - // params.Extra - BigInt - sector id - // params.Proof - nil - - _, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } - - // TODO: ensure no sector ID reusability within related deal lifetime - sector := types.BigFromBytes(params.Extra) - - if len(params.Proof) > 0 { - return nil, aerrors.New(1, "unexpected proof bytes") - } - - ok, _, _, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, sector.Uint64()) - if aerr != nil { - return nil, aerr - } - if !ok { - return nil, aerrors.New(2, "miner does not have required sector") - } - - return nil, nil -} - type AddFaultsParams struct { Faults types.BitField } diff --git a/chain/actors/actor_multisig_test.go b/chain/actors/actor_multisig_test.go index b705ad4a8..b55d17b13 100644 --- a/chain/actors/actor_multisig_test.go +++ b/chain/actors/actor_multisig_test.go @@ -23,7 +23,7 @@ func TestMultiSigCreate(t *testing.T) { } h := NewHarness(t, opts...) - ret, _ := h.CreateActor(t, creatorAddr, actors.MultisigActorCodeCid, + ret, _ := h.CreateActor(t, creatorAddr, actors.MultisigCodeCid, &actors.MultiSigConstructorParams{ Signers: []address.Address{creatorAddr, sig1Addr, sig2Addr}, Required: 2, @@ -49,7 +49,7 @@ func TestMultiSigOps(t *testing.T) { HarnessAddr(&sig1Addr, 100000), HarnessAddr(&sig2Addr, 100000), HarnessAddr(&outsideAddr, 100000), - HarnessActor(&multSigAddr, &creatorAddr, actors.MultisigActorCodeCid, + HarnessActor(&multSigAddr, &creatorAddr, actors.MultisigCodeCid, func() cbg.CBORMarshaler { return &actors.MultiSigConstructorParams{ Signers: []address.Address{creatorAddr, sig1Addr, sig2Addr}, diff --git a/chain/actors/actor_paych_test.go b/chain/actors/actor_paych_test.go index 3b712598c..e23acdf4a 100644 --- a/chain/actors/actor_paych_test.go +++ b/chain/actors/actor_paych_test.go @@ -18,7 +18,7 @@ func TestPaychCreate(t *testing.T) { } h := NewHarness(t, opts...) - ret, _ := h.CreateActor(t, creatorAddr, actors.PaymentChannelActorCodeCid, + ret, _ := h.CreateActor(t, creatorAddr, actors.PaymentChannelCodeCid, &actors.PCAConstructorParams{ To: targetAddr, }) @@ -47,7 +47,7 @@ func TestPaychUpdate(t *testing.T) { } h := NewHarness(t, opts...) - ret, _ := h.CreateActor(t, creatorAddr, actors.PaymentChannelActorCodeCid, + ret, _ := h.CreateActor(t, creatorAddr, actors.PaymentChannelCodeCid, &actors.PCAConstructorParams{ To: targetAddr, }) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 6004c6113..3753621d4 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -2,16 +2,17 @@ package actors import ( "bytes" - + "context" "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" - "github.com/filecoin-project/go-lotus/build" - "github.com/filecoin-project/go-lotus/chain/actors/aerrors" - "github.com/filecoin-project/go-lotus/chain/address" - "github.com/filecoin-project/go-lotus/chain/types" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" ) type StorageMarketActor struct{} @@ -59,16 +60,64 @@ type StorageMarketState struct { NextDealID uint64 // TODO: amt.LastIndex() } +// TODO: serialization mode spec +type SerializationMode uint64 + +const ( + SerializationUnixFSv0 = iota + // IPLD / car +) + type StorageDealProposal struct { PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea PieceSize uint64 - Client address.Address - Provider address.Address + PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + + Client address.Address + Provider address.Address + ProposalExpiration uint64 - DealExpiration uint64 - StoragePrice types.BigInt - StorageCollateral types.BigInt - ProposerSignature types.Signature + Duration uint64 // TODO: spec proposes 'DealExpiration', but that's awkward as it + // doesn't tell when the deal actually starts, so the price per block is impossible to + // calculate. It also doesn't incentivize the miner to seal / activate sooner, as he + // still get's paid the full amount specified in the deal + // + // Changing to duration makes sure that the price-per-block is defined, and the miner + // doesn't get paid when not storing the sector + + StoragePrice types.BigInt + StorageCollateral types.BigInt + + ProposerSignature *types.Signature +} + +type SignFunc = func(context.Context, []byte) (*types.Signature, error) + +func (sdp *StorageDealProposal) Sign(ctx context.Context, sign SignFunc) error { + if sdp.ProposerSignature != nil { + return xerrors.New("signature already present in StorageDealProposal") + } + var buf bytes.Buffer + if err := sdp.MarshalCBOR(&buf); err != nil { + return err + } + sig, err := sign(ctx, buf.Bytes()) + if err != nil { + return err + } + sdp.ProposerSignature = sig + return nil +} + +func (sdp *StorageDealProposal) Verify() error { + unsigned := *sdp + unsigned.ProposerSignature = nil + var buf bytes.Buffer + if err := sdp.MarshalCBOR(&buf); err != nil { + return err + } + + return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) } type StorageDeal struct { @@ -86,6 +135,8 @@ type WithdrawBalanceParams struct { } func (sma StorageMarketActor) WithdrawBalance(act *types.Actor, vmctx types.VMContext, params *WithdrawBalanceParams) ([]byte, ActorError) { + // TODO: (spec) this should be 2-stage + var self StorageMarketState old := vmctx.Storage().GetHead() if err := vmctx.Storage().Get(old, &self); err != nil { @@ -527,10 +578,6 @@ func (sma StorageMarketActor) SettleExpiredDeals(act *types.Actor, vmctx types.V } -func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { - -} - func (sma StorageMarketActor) SlashStorageDealCollateral(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { } diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index 21afb6dd0..a8e47ea2a 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -87,7 +87,7 @@ func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VM return nil, err } - ret, err := vmctx.Send(InitActorAddress, IAMethods.Exec, vmctx.Message().Value, encoded) + ret, err := vmctx.Send(InitAddress, IAMethods.Exec, vmctx.Message().Value, encoded) if err != nil { return nil, err } diff --git a/chain/actors/actors.go b/chain/actors/actors.go index 0f18964cb..032c6e573 100644 --- a/chain/actors/actors.go +++ b/chain/actors/actors.go @@ -7,15 +7,15 @@ import ( mh "github.com/multiformats/go-multihash" ) -var AccountActorCodeCid cid.Cid -var StoragePowerActorCodeCid cid.Cid -var StorageMarketActorCodeCid cid.Cid +var AccountCodeCid cid.Cid +var StoragePowerCodeCid cid.Cid +var StorageMarketCodeCid cid.Cid var StorageMinerCodeCid cid.Cid -var MultisigActorCodeCid cid.Cid -var InitActorCodeCid cid.Cid -var PaymentChannelActorCodeCid cid.Cid +var MultisigCodeCid cid.Cid +var InitCodeCid cid.Cid +var PaymentChannelCodeCid cid.Cid -var InitActorAddress = mustIDAddress(0) +var InitAddress = mustIDAddress(0) var NetworkAddress = mustIDAddress(1) var StoragePowerAddress = mustIDAddress(2) var StorageMarketAddress = mustIDAddress(3) // TODO: missing from spec @@ -39,11 +39,11 @@ func init() { return c } - AccountActorCodeCid = mustSum("filecoin/1.0/AccountActor") - StoragePowerActorCodeCid = mustSum("filecoin/1.0/StoragePowerActor") - StorageMarketActorCodeCid = mustSum("filecoin/1.0/StorageMarketActor") + AccountCodeCid = mustSum("filecoin/1.0/AccountActor") + StoragePowerCodeCid = mustSum("filecoin/1.0/StoragePowerActor") + StorageMarketCodeCid = mustSum("filecoin/1.0/StorageMarketActor") StorageMinerCodeCid = mustSum("filecoin/1.0/StorageMinerActor") - MultisigActorCodeCid = mustSum("filecoin/1.0/MultisigActor") - InitActorCodeCid = mustSum("filecoin/1.0/InitActor") - PaymentChannelActorCodeCid = mustSum("filecoin/1.0/PaymentChannelActor") + MultisigCodeCid = mustSum("filecoin/1.0/MultisigActor") + InitCodeCid = mustSum("filecoin/1.0/InitActor") + PaymentChannelCodeCid = mustSum("filecoin/1.0/PaymentChannelActor") } diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 27946664c..57d559cf8 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -80,7 +80,7 @@ func TestVMInvokeMethod(t *testing.T) { } msg := &types.Message{ - To: InitActorAddress, + To: InitAddress, From: from, Method: IAMethods.Exec, Params: enc, diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index f0fc4b3e7..0ffc02b1b 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -210,7 +210,7 @@ func (h *Harness) CreateActor(t testing.TB, from address.Address, t.Helper() return h.Apply(t, types.Message{ - To: actors.InitActorAddress, + To: actors.InitAddress, From: from, Method: actors.IAMethods.Exec, Params: DumpObject(t, diff --git a/chain/deals/client.go b/chain/deals/client.go index bf059ab93..8e889836d 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/chain/store" "math" "github.com/ipfs/go-cid" @@ -27,20 +28,16 @@ import ( func init() { cbor.RegisterCborType(ClientDeal{}) - cbor.RegisterCborType(actors.PieceInclVoucherData{}) // TODO: USE CBORGEN! cbor.RegisterCborType(types.SignedVoucher{}) cbor.RegisterCborType(types.ModVerifyParams{}) cbor.RegisterCborType(types.Signature{}) - cbor.RegisterCborType(actors.PaymentInfo{}) - cbor.RegisterCborType(api.PaymentInfo{}) - cbor.RegisterCborType(actors.InclusionProof{}) } var log = logging.Logger("deals") type ClientDeal struct { ProposalCid cid.Cid - Proposal StorageDealProposal + Proposal actors.StorageDealProposal State api.DealState Miner peer.ID @@ -49,6 +46,7 @@ type ClientDeal struct { type Client struct { sm *stmgr.StateManager + chain *store.ChainStore h host.Host w *wallet.Wallet dag dtypes.ClientDAG @@ -70,9 +68,10 @@ type clientDealUpdate struct { err error } -func NewClient(sm *stmgr.StateManager, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { c := &Client{ sm: sm, + chain: chain, h: h, w: w, dag: dag, @@ -164,49 +163,32 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { } type ClientDealProposal struct { - Data cid.Cid + Data cid.Cid + DataSize uint64 - TotalPrice types.BigInt - Duration uint64 + TotalPrice types.BigInt + ProposalExpiration uint64 + Duration uint64 - Payment actors.PaymentInfo - - MinerAddress address.Address - ClientAddress address.Address - MinerID peer.ID + ProviderAddress address.Address + Client address.Address + MinerID peer.ID } -func (c *Client) VerifyParams(ctx context.Context, data cid.Cid) (*actors.PieceInclVoucherData, error) { - commP, size, err := c.commP(ctx, data) - if err != nil { - return nil, err +func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { + proposal := actors.StorageDealProposal{ + PieceRef: p.Data.Bytes(), + PieceSize: p.DataSize, + PieceSerialization: actors.SerializationUnixFSv0, + Client: p.Client, + Provider: p.ProviderAddress, + ProposalExpiration: p.ProposalExpiration, + Duration: p.Duration, + StoragePrice: p.TotalPrice, + StorageCollateral: types.NewInt(p.DataSize), // TODO: real calc } - return &actors.PieceInclVoucherData{ - CommP: commP, - PieceSize: types.NewInt(uint64(size)), - }, nil -} - -func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.PieceInclVoucherData) (cid.Cid, error) { - proposal := StorageDealProposal{ - PieceRef: p.Data, - SerializationMode: SerializationUnixFs, - CommP: vd.CommP[:], - Size: vd.PieceSize.Uint64(), - TotalPrice: p.TotalPrice, - Duration: p.Duration, - Payment: p.Payment, - MinerAddress: p.MinerAddress, - ClientAddress: p.ClientAddress, - } - - s, err := c.h.NewStream(ctx, p.MinerID, ProtocolID) - if err != nil { - return cid.Undef, err - } - - if err := c.sendProposal(s, proposal, p.ClientAddress); err != nil { + if err := api.SignWith(ctx, c.w.Sign, p.Client, &proposal); err != nil { return cid.Undef, err } @@ -215,6 +197,17 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.Pie return cid.Undef, err } + s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) + if err != nil { + s.Reset() + return cid.Undef, err + } + + if err := cborrpc.WriteCborRPC(s, proposal); err != nil { + s.Reset() + return cid.Undef, err + } + deal := ClientDeal{ ProposalCid: proposalNd.Cid(), Proposal: proposal, @@ -224,12 +217,10 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal, vd *actors.Pie s: s, } - // TODO: actually care about what happens with the deal after it was accepted c.incoming <- deal - // TODO: start tracking after the deal is sealed return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ - Address: proposal.MinerAddress, + Address: proposal.Provider, ID: deal.Miner, }) } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index d19270707..28164c61f 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -3,11 +3,9 @@ package deals import ( "context" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error @@ -39,6 +37,34 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) error { return xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) } + // TODO: spec says it's optional + pubmsg, err := c.chain.GetMessage(resp.PublishMessage) + if err != nil { + return xerrors.Errorf("getting deal pubsish message: %w", err) + } + + if pubmsg.From != deal.Proposal.Provider { + return xerrors.Errorf("Deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + } + + if pubmsg.To != actors.StorageMarketAddress { + return xerrors.Errorf("Deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + } + + if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { + return xerrors.Errorf("Deal publish message called incorrect method (method=%s)", pubmsg.Method) + } + + // TODO: timeout + _, ret, err := c.sm.WaitForMessage(ctx, resp.PublishMessage) + if err != nil { + return xerrors.Errorf("Waiting for deal publish message: %w", err) + } + if ret.ExitCode != 0 { + return xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) + } + // TODO: persist dealId + log.Info("DEAL ACCEPTED!") return nil @@ -75,13 +101,14 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) error { log.Info("DEAL SEALED!") - ok, err := sectorbuilder.VerifyPieceInclusionProof(build.SectorSize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) + // TODO: want? + /*ok, err := sectorbuilder.VerifyPieceInclusionProof(build.SectorSize, deal.Proposal.PieceSize, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) if err != nil { return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err) } if !ok { return xerrors.Errorf("verifying piece inclusion proof in staged deal %s failed", deal.ProposalCid) - } + }*/ return nil } diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index c192bbeb4..648e41733 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -1,18 +1,11 @@ package deals import ( - "context" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "runtime" "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-files" - cbor "github.com/ipfs/go-ipld-cbor" - unixfile "github.com/ipfs/go-unixfs/file" - inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/cborrpc" ) @@ -32,58 +25,6 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) } -func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, int64, error) { - root, err := c.dag.Get(ctx, data) - if err != nil { - log.Errorf("failed to get file root for deal: %s", err) - return nil, 0, err - } - - n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) - if err != nil { - log.Errorf("cannot open unixfs file: %s", err) - return nil, 0, err - } - - uf, ok := n.(files.File) - if !ok { - // TODO: we probably got directory, how should we handle this in unixfs mode? - return nil, 0, xerrors.New("unsupported unixfs type") - } - - size, err := uf.Size() - if err != nil { - return nil, 0, err - } - - commP, err := sectorbuilder.GeneratePieceCommitment(uf, uint64(size)) - if err != nil { - return nil, 0, err - } - - return commP[:], size, err -} - -func (c *Client) sendProposal(s inet.Stream, proposal StorageDealProposal, from address.Address) error { - log.Info("Sending deal proposal") - - msg, err := cbor.DumpObject(proposal) - if err != nil { - return err - } - sig, err := c.w.Sign(context.TODO(), from, msg) - if err != nil { - return err - } - - signedProposal := &SignedStorageDealProposal{ - Proposal: proposal, - Signature: sig, - } - - return cborrpc.WriteCborRPC(s, signedProposal) -} - func (c *Client) readStorageDealResp(deal ClientDeal) (*StorageDealResponse, error) { s, ok := c.conns[deal.ProposalCid] if !ok { diff --git a/chain/deals/handler_states.go b/chain/deals/handler_states.go deleted file mode 100644 index a981497e3..000000000 --- a/chain/deals/handler_states.go +++ /dev/null @@ -1,313 +0,0 @@ -package deals - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-sectorbuilder/sealing_state" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/ipfs/go-merkledag" - unixfile "github.com/ipfs/go-unixfs/file" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/storage/sectorblocks" -) - -type minerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) - -func (h *Handler) handle(ctx context.Context, deal MinerDeal, cb minerHandlerFunc, next api.DealState) { - go func() { - mut, err := cb(ctx, deal) - - if err == nil && next == api.DealNoUpdate { - return - } - - select { - case h.updated <- minerDealUpdate{ - newState: next, - id: deal.ProposalCid, - err: err, - mut: mut, - }: - case <-h.stop: - } - }() -} - -// ACCEPTED - -func (h *Handler) checkVoucher(ctx context.Context, deal MinerDeal, voucher *types.SignedVoucher, lane uint64, maxClose uint64, amount types.BigInt) error { - err := h.full.PaychVoucherCheckValid(ctx, deal.Proposal.Payment.PayChActor, voucher) - if err != nil { - return err - } - - if voucher.Extra == nil { - return xerrors.New("voucher.Extra not set") - } - - if voucher.Extra.Actor != deal.Proposal.MinerAddress { - return xerrors.Errorf("extra params actor didn't match miner address in proposal: '%s' != '%s'", voucher.Extra.Actor, deal.Proposal.MinerAddress) - } - - if voucher.Extra.Method != actors.MAMethods.PaymentVerifyInclusion { - return xerrors.Errorf("expected extra method %d, got %d", actors.MAMethods.PaymentVerifyInclusion, voucher.Extra.Method) - } - - var inclChallenge actors.PieceInclVoucherData - if err := cbor.DecodeInto(voucher.Extra.Data, &inclChallenge); err != nil { - return xerrors.Errorf("failed to decode storage voucher data for verification: %w", err) - } - - if inclChallenge.PieceSize.Uint64() != deal.Proposal.Size { - return xerrors.Errorf("paych challenge piece size didn't match deal proposal size: %d != %d", inclChallenge.PieceSize.Uint64(), deal.Proposal.Size) - } - - if !bytes.Equal(inclChallenge.CommP, deal.Proposal.CommP) { - return xerrors.New("paych challenge commP didn't match deal proposal") - } - - if voucher.MinCloseHeight > maxClose { - return xerrors.Errorf("MinCloseHeight too high (%d), max expected: %d", voucher.MinCloseHeight, maxClose) - } - - if voucher.TimeLock > maxClose { - return xerrors.Errorf("TimeLock too high (%d), max expected: %d", voucher.TimeLock, maxClose) - } - - if len(voucher.Merges) > 0 { - return xerrors.New("didn't expect any merges") - } - - if voucher.Amount.LessThan(amount) { - return xerrors.Errorf("not enough funds in the voucher: %s < %s; vl=%d", voucher.Amount, amount, len(deal.Proposal.Payment.Vouchers)) - } - - if voucher.Lane != lane { - return xerrors.Errorf("expected all vouchers on lane %d, found voucher on lane %d", lane, voucher.Lane) - } - - return nil -} - -func (h *Handler) consumeVouchers(ctx context.Context, deal MinerDeal) error { - curHead, err := h.full.ChainHead(ctx) - if err != nil { - return err - } - if len(deal.Proposal.Payment.Vouchers) == 0 { - return xerrors.Errorf("no payment vouchers for deal") - } - - increment := deal.Proposal.Duration / uint64(len(deal.Proposal.Payment.Vouchers)) - - startH := deal.Proposal.Payment.Vouchers[0].TimeLock - increment - if startH > curHead.Height()+build.DealVoucherSkewLimit { - return xerrors.Errorf("deal starts too far into the future: start=%d; h=%d; max=%d; inc=%d", startH, curHead.Height(), curHead.Height()+build.DealVoucherSkewLimit, increment) - } - - vspec := VoucherSpec(deal.Proposal.Duration, deal.Proposal.TotalPrice, startH, nil) - - lane := deal.Proposal.Payment.Vouchers[0].Lane - - for i, voucher := range deal.Proposal.Payment.Vouchers { - maxClose := curHead.Height() + (increment * uint64(i+1)) + build.DealVoucherSkewLimit - - if err := h.checkVoucher(ctx, deal, voucher, lane, maxClose, vspec[i].Amount); err != nil { - return xerrors.Errorf("validating payment voucher %d: %w", i, err) - } - } - - minPrice := types.BigMul(types.BigMul(h.pricePerByteBlock, types.NewInt(deal.Proposal.Size)), types.NewInt(deal.Proposal.Duration)) - if types.BigCmp(minPrice, deal.Proposal.TotalPrice) > 0 { - return xerrors.Errorf("minimum price: %s", minPrice) - } - - prevAmt := types.NewInt(0) - - for i, voucher := range deal.Proposal.Payment.Vouchers { - delta, err := h.full.PaychVoucherAdd(ctx, deal.Proposal.Payment.PayChActor, voucher, nil, types.BigSub(vspec[i].Amount, prevAmt)) - if err != nil { - return xerrors.Errorf("consuming payment voucher %d: %w", i, err) - } - prevAmt = types.BigAdd(prevAmt, delta) - } - - return nil -} - -func (h *Handler) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - switch deal.Proposal.SerializationMode { - //case SerializationRaw: - //case SerializationIPLD: - case SerializationUnixFs: - default: - return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.SerializationMode) - } - - if deal.Proposal.Payment.ChannelMessage != nil { - log.Info("waiting for channel message to appear on chain") - if _, err := h.full.StateWaitMsg(ctx, *deal.Proposal.Payment.ChannelMessage); err != nil { - return nil, xerrors.Errorf("waiting for paych message: %w", err) - } - } - - if err := h.consumeVouchers(ctx, deal); err != nil { - return nil, err - } - - log.Info("fetching data for a deal") - err := h.sendSignedResponse(StorageDealResponse{ - State: api.DealAccepted, - Message: "", - Proposal: deal.ProposalCid, - }) - if err != nil { - return nil, err - } - - return nil, merkledag.FetchGraph(ctx, deal.Ref, h.dag) -} - -// STAGED - -func (h *Handler) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := h.sendSignedResponse(StorageDealResponse{ - State: api.DealStaged, - Proposal: deal.ProposalCid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - - root, err := h.dag.Get(ctx, deal.Ref) - if err != nil { - return nil, xerrors.Errorf("failed to get file root for deal: %s", err) - } - - // TODO: abstract this away into ReadSizeCloser + implement different modes - n, err := unixfile.NewUnixfsFile(ctx, h.dag, root) - if err != nil { - return nil, xerrors.Errorf("cannot open unixfs file: %s", err) - } - - uf, ok := n.(sectorblocks.UnixfsReader) - if !ok { - // we probably got directory, unsupported for now - return nil, xerrors.Errorf("unsupported unixfs file type") - } - - sectorID, err := h.secst.AddUnixfsPiece(deal.Proposal.PieceRef, uf, deal.Proposal.Duration) - if err != nil { - return nil, xerrors.Errorf("AddPiece failed: %s", err) - } - - log.Warnf("New Sector: %d", sectorID) - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil -} - -// SEALING - -func getInclusionProof(ref string, status sectorbuilder.SectorSealingStatus) (PieceInclusionProof, error) { - for i, p := range status.Pieces { - if p.Key == ref { - return PieceInclusionProof{ - Position: uint64(i), - ProofElements: p.InclusionProof, - }, nil - } - } - return PieceInclusionProof{}, xerrors.Errorf("pieceInclusionProof for %s in sector %d not found", ref, status.SectorID) -} - -func (h *Handler) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) { - status, err := h.secst.WaitSeal(ctx, deal.SectorID) - if err != nil { - return sectorbuilder.SectorSealingStatus{}, err - } - - switch status.State { - case sealing_state.Sealed: - case sealing_state.Failed: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) - case sealing_state.Pending: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) - case sealing_state.Sealing: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) - default: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) - } - - return status, nil -} - -func (h *Handler) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - status, err := h.waitSealed(ctx, deal) - if err != nil { - return nil, err - } - - // TODO: don't hardcode unixfs - ip, err := getInclusionProof(string(sectorblocks.SerializationUnixfs0)+deal.Ref.String(), status) - if err != nil { - return nil, err - } - - proof := &actors.InclusionProof{ - Sector: deal.SectorID, - Proof: ip.ProofElements, - } - proofB, err := cbor.DumpObject(proof) - if err != nil { - return nil, err - } - - // store proofs for channels - for i, v := range deal.Proposal.Payment.Vouchers { - if v.Extra.Method == actors.MAMethods.PaymentVerifyInclusion { - // TODO: Set correct minAmount - if _, err := h.full.PaychVoucherAdd(ctx, deal.Proposal.Payment.PayChActor, v, proofB, types.NewInt(0)); err != nil { - return nil, xerrors.Errorf("storing payment voucher %d proof: %w", i, err) - } - } - } - - err = h.sendSignedResponse(StorageDealResponse{ - State: api.DealSealing, - Proposal: deal.ProposalCid, - PieceInclusionProof: ip, - CommD: status.CommD[:], - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - - return nil, nil -} - -func (h *Handler) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - mcid, err := h.commt.WaitCommit(ctx, deal.Proposal.MinerAddress, deal.SectorID) - if err != nil { - log.Warnf("Waiting for sector commitment message: %s", err) - } - - err = h.sendSignedResponse(StorageDealResponse{ - State: api.DealComplete, - Proposal: deal.ProposalCid, - - SectorCommitMessage: &mcid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - - return nil, nil -} diff --git a/chain/deals/handler.go b/chain/deals/provider.go similarity index 71% rename from chain/deals/handler.go rename to chain/deals/provider.go index 02db7153c..67fbf33f1 100644 --- a/chain/deals/handler.go +++ b/chain/deals/provider.go @@ -14,6 +14,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -27,7 +28,7 @@ func init() { type MinerDeal struct { Client peer.ID - Proposal StorageDealProposal + Proposal actors.StorageDealProposal ProposalCid cid.Cid State api.DealState @@ -38,7 +39,7 @@ type MinerDeal struct { s inet.Stream } -type Handler struct { +type Provider struct { pricePerByteBlock types.BigInt // how much we want for storing one byte for one block minPieceSize uint64 @@ -73,7 +74,7 @@ type minerDealUpdate struct { mut func(*MinerDeal) } -func NewHandler(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *commitment.Tracker, dag dtypes.StagingDAG, fullNode api.FullNode) (*Handler, error) { +func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *commitment.Tracker, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -83,7 +84,7 @@ func NewHandler(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *c return nil, err } - h := &Handler{ + h := &Provider{ secst: secst, commt: commt, dag: dag, @@ -120,40 +121,40 @@ func NewHandler(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *c return h, nil } -func (h *Handler) Run(ctx context.Context) { +func (p *Provider) Run(ctx context.Context) { // TODO: restore state go func() { - defer log.Warn("quitting deal handler loop") - defer close(h.stopped) + defer log.Warn("quitting deal provider loop") + defer close(p.stopped) for { select { - case deal := <-h.incoming: // DealAccepted - h.onIncoming(deal) - case update := <-h.updated: // DealStaged - h.onUpdated(ctx, update) - case <-h.stop: + case deal := <-p.incoming: // DealAccepted + p.onIncoming(deal) + case update := <-p.updated: // DealStaged + p.onUpdated(ctx, update) + case <-p.stop: return } } }() } -func (h *Handler) onIncoming(deal MinerDeal) { +func (p *Provider) onIncoming(deal MinerDeal) { log.Info("incoming deal") - h.conns[deal.ProposalCid] = deal.s + p.conns[deal.ProposalCid] = deal.s - if err := h.deals.Begin(deal.ProposalCid, deal); err != nil { + if err := p.deals.Begin(deal.ProposalCid, deal); err != nil { // This can happen when client re-sends proposal - h.failDeal(deal.ProposalCid, err) + p.failDeal(deal.ProposalCid, err) log.Errorf("deal tracking failed: %s", err) return } go func() { - h.updated <- minerDealUpdate{ + p.updated <- minerDealUpdate{ newState: api.DealAccepted, id: deal.ProposalCid, err: nil, @@ -161,15 +162,15 @@ func (h *Handler) onIncoming(deal MinerDeal) { }() } -func (h *Handler) onUpdated(ctx context.Context, update minerDealUpdate) { +func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { log.Infof("Deal %s updated state to %d", update.id, update.newState) if update.err != nil { log.Errorf("deal %s failed: %s", update.id, update.err) - h.failDeal(update.id, update.err) + p.failDeal(update.id, update.err) return } var deal MinerDeal - err := h.deals.MutateMiner(update.id, func(d *MinerDeal) error { + err := p.deals.MutateMiner(update.id, func(d *MinerDeal) error { d.State = update.newState if update.mut != nil { update.mut(d) @@ -178,30 +179,29 @@ func (h *Handler) onUpdated(ctx context.Context, update minerDealUpdate) { return nil }) if err != nil { - h.failDeal(update.id, err) + p.failDeal(update.id, err) return } switch update.newState { case api.DealAccepted: - h.handle(ctx, deal, h.accept, api.DealStaged) + p.handle(ctx, deal, p.accept, api.DealStaged) case api.DealStaged: - h.handle(ctx, deal, h.staged, api.DealSealing) + p.handle(ctx, deal, p.staged, api.DealSealing) case api.DealSealing: - h.handle(ctx, deal, h.sealing, api.DealComplete) + p.handle(ctx, deal, p.sealing, api.DealComplete) case api.DealComplete: - h.handle(ctx, deal, h.complete, api.DealNoUpdate) + p.handle(ctx, deal, p.complete, api.DealNoUpdate) } } -func (h *Handler) newDeal(s inet.Stream, proposal StorageDealProposal) (MinerDeal, error) { - // TODO: Review: Not signed? +func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) { proposalNd, err := cbor.WrapObject(proposal, math.MaxUint64, -1) if err != nil { return MinerDeal{}, err } - ref, err := cid.Parse(proposal.PieceRef) + ref, err := cid.Cast(proposal.PieceRef) if err != nil { return MinerDeal{}, err } @@ -218,27 +218,27 @@ func (h *Handler) newDeal(s inet.Stream, proposal StorageDealProposal) (MinerDea }, nil } -func (h *Handler) HandleStream(s inet.Stream) { +func (p *Provider) HandleStream(s inet.Stream) { log.Info("Handling storage deal proposal!") - proposal, err := h.readProposal(s) + proposal, err := p.readProposal(s) if err != nil { log.Error(err) s.Close() return } - deal, err := h.newDeal(s, proposal.Proposal) + deal, err := p.newDeal(s, proposal) if err != nil { log.Error(err) s.Close() return } - h.incoming <- deal + p.incoming <- deal } -func (h *Handler) Stop() { - close(h.stop) - <-h.stopped +func (p *Provider) Stop() { + close(p.stop) + <-p.stopped } diff --git a/chain/deals/asks.go b/chain/deals/provider_asks.go similarity index 66% rename from chain/deals/asks.go rename to chain/deals/provider_asks.go index df82550e8..af7c8cce6 100644 --- a/chain/deals/asks.go +++ b/chain/deals/provider_asks.go @@ -14,44 +14,44 @@ import ( "golang.org/x/xerrors" ) -func (h *Handler) SetPrice(p types.BigInt, ttlsecs int64) error { - h.askLk.Lock() - defer h.askLk.Unlock() +func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { + p.askLk.Lock() + defer p.askLk.Unlock() var seqno uint64 - if h.ask != nil { - seqno = h.ask.Ask.SeqNo + 1 + if p.ask != nil { + seqno = p.ask.Ask.SeqNo + 1 } now := time.Now().Unix() ask := &types.StorageAsk{ - Price: p, + Price: price, Timestamp: now, Expiry: now + ttlsecs, - Miner: h.actor, + Miner: p.actor, SeqNo: seqno, - MinPieceSize: h.minPieceSize, + MinPieceSize: p.minPieceSize, } - ssa, err := h.signAsk(ask) + ssa, err := p.signAsk(ask) if err != nil { return err } - return h.saveAsk(ssa) + return p.saveAsk(ssa) } -func (h *Handler) getAsk(m address.Address) *types.SignedStorageAsk { - h.askLk.Lock() - defer h.askLk.Unlock() - if m != h.actor { +func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk { + p.askLk.Lock() + defer p.askLk.Unlock() + if m != p.actor { return nil } - return h.ask + return p.ask } -func (h *Handler) HandleAskStream(s inet.Stream) { +func (p *Provider) HandleAskStream(s inet.Stream) { defer s.Close() var ar AskRequest if err := cborrpc.ReadCborRPC(s, &ar); err != nil { @@ -59,7 +59,7 @@ func (h *Handler) HandleAskStream(s inet.Stream) { return } - resp := h.processAskRequest(&ar) + resp := p.processAskRequest(&ar) if err := cborrpc.WriteCborRPC(s, resp); err != nil { log.Errorf("failed to write ask response: %s", err) @@ -67,19 +67,19 @@ func (h *Handler) HandleAskStream(s inet.Stream) { } } -func (h *Handler) processAskRequest(ar *AskRequest) *AskResponse { +func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse { return &AskResponse{ - Ask: h.getAsk(ar.Miner), + Ask: p.getAsk(ar.Miner), } } var bestAskKey = datastore.NewKey("latest-ask") -func (h *Handler) tryLoadAsk() error { - h.askLk.Lock() - defer h.askLk.Unlock() +func (p *Provider) tryLoadAsk() error { + p.askLk.Lock() + defer p.askLk.Unlock() - err := h.loadAsk() + err := p.loadAsk() if err != nil { if xerrors.Is(err, datastore.ErrNotFound) { log.Warn("no previous ask found, miner will not accept deals until a price is set") @@ -91,8 +91,8 @@ func (h *Handler) tryLoadAsk() error { return nil } -func (h *Handler) loadAsk() error { - askb, err := h.ds.Get(datastore.NewKey("latest-ask")) +func (p *Provider) loadAsk() error { + askb, err := p.ds.Get(datastore.NewKey("latest-ask")) if err != nil { return xerrors.Errorf("failed to load most recent ask from disk: %w", err) } @@ -102,22 +102,22 @@ func (h *Handler) loadAsk() error { return err } - h.ask = &ssa + p.ask = &ssa return nil } -func (h *Handler) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { +func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { b, err := cbor.DumpObject(a) if err != nil { return nil, err } - worker, err := h.getWorker(h.actor) + worker, err := p.getWorker(p.actor) if err != nil { return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err) } - sig, err := h.full.WalletSign(context.TODO(), worker, b) + sig, err := p.full.WalletSign(context.TODO(), worker, b) if err != nil { return nil, err } @@ -128,17 +128,17 @@ func (h *Handler) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) }, nil } -func (h *Handler) saveAsk(a *types.SignedStorageAsk) error { +func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { b, err := cbor.DumpObject(a) if err != nil { return err } - if err := h.ds.Put(bestAskKey, b); err != nil { + if err := p.ds.Put(bestAskKey, b); err != nil { return err } - h.ask = a + p.ask = a return nil } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go new file mode 100644 index 000000000..a7c321faa --- /dev/null +++ b/chain/deals/provider_states.go @@ -0,0 +1,245 @@ +package deals + +import ( + "context" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-sectorbuilder/sealing_state" + "github.com/ipfs/go-merkledag" + unixfile "github.com/ipfs/go-unixfs/file" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/storage/sectorblocks" +) + +type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) + +func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) { + go func() { + mut, err := cb(ctx, deal) + + if err == nil && next == api.DealNoUpdate { + return + } + + select { + case p.updated <- minerDealUpdate{ + newState: next, + id: deal.ProposalCid, + err: err, + mut: mut, + }: + case <-p.stop: + } + }() +} + +// ACCEPTED + +func (p *Provider) addMarketFunds(ctx context.Context, deal MinerDeal) error { + log.Info("Adding market funds for storage collateral") + smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: deal.Proposal.Provider, + Value: deal.Proposal.StorageCollateral, + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.Receipt.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) + } + + return nil +} + +func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { + switch deal.Proposal.PieceSerialization { + //case SerializationRaw: + //case SerializationIPLD: + case actors.SerializationUnixFSv0: + default: + return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.PieceSerialization) + } + + // TODO: check StorageCollateral / StoragePrice + + // check market funds + clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client) + if err != nil { + return nil, err + } + + // This doesn't guarantee that the client won't withdraw / lock those funds + // but it's a decent first filter + if clientMarketBalance.Available.LessThan(deal.Proposal.StoragePrice) { + return nil, xerrors.New("clientMarketBalance.Available too small") + } + + providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client) + if err != nil { + return nil, err + } + + // TODO: this needs to be atomic + if providerMarketBalance.Available.LessThan(deal.Proposal.StorageCollateral) { + if err := p.addMarketFunds(ctx, deal); err != nil { + return nil, err + } + } + + log.Info("publishing deal") + + // TODO: We may want this to happen after fetching data + smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: deal.Proposal.Provider, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + }) + if err != nil { + return nil, err + } + r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return nil, err + } + if r.Receipt.ExitCode != 0 { + return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) + } + + log.Info("fetching data for a deal") + err = p.sendSignedResponse(StorageDealResponse{ + State: api.DealAccepted, + Message: "", + Proposal: deal.ProposalCid, + PublishMessage: smsg.Cid(), + }) + if err != nil { + return nil, err + } + + return nil, merkledag.FetchGraph(ctx, deal.Ref, p.dag) +} + +// STAGED + +func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { + err := p.sendSignedResponse(StorageDealResponse{ + State: api.DealStaged, + Proposal: deal.ProposalCid, + }) + if err != nil { + log.Warnf("Sending deal response failed: %s", err) + } + + root, err := p.dag.Get(ctx, deal.Ref) + if err != nil { + return nil, xerrors.Errorf("failed to get file root for deal: %s", err) + } + + // TODO: abstract this away into ReadSizeCloser + implement different modes + n, err := unixfile.NewUnixfsFile(ctx, p.dag, root) + if err != nil { + return nil, xerrors.Errorf("cannot open unixfs file: %s", err) + } + + uf, ok := n.(sectorblocks.UnixfsReader) + if !ok { + // we probably got directory, unsupported for now + return nil, xerrors.Errorf("unsupported unixfs file type") + } + + pcid, err := cid.Cast(deal.Proposal.PieceRef) + if err != nil { + return nil, err + } + + sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.Proposal.Duration) + if err != nil { + return nil, xerrors.Errorf("AddPiece failed: %s", err) + } + + log.Warnf("New Sector: %d", sectorID) + return func(deal *MinerDeal) { + deal.SectorID = sectorID + }, nil +} + +// SEALING + +func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) { + status, err := p.secst.WaitSeal(ctx, deal.SectorID) + if err != nil { + return sectorbuilder.SectorSealingStatus{}, err + } + + switch status.State { + case sealing_state.Sealed: + case sealing_state.Failed: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) + case sealing_state.Pending: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) + case sealing_state.Sealing: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) + default: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) + } + + return status, nil +} + +func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { + err := p.sendSignedResponse(StorageDealResponse{ + State: api.DealSealing, + Proposal: deal.ProposalCid, + }) + if err != nil { + log.Warnf("Sending deal response failed: %s", err) + } + + _, err = p.waitSealed(ctx, deal) + if err != nil { + return nil, err + } + // TODO: Spec doesn't say anything about inclusion proofs anywhere + // Not sure what mechanisms prevents miner from storing data that isn't + // clients' data + + return nil, nil +} + +func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { + // TODO: Add dealID to commtracker (probably before sealing) + mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) + if err != nil { + log.Warnf("Waiting for sector commitment message: %s", err) + } + + err = p.sendSignedResponse(StorageDealResponse{ + State: api.DealComplete, + Proposal: deal.ProposalCid, + + CommitMessage: mcid, + }) + if err != nil { + log.Warnf("Sending deal response failed: %s", err) + } + + return nil, nil +} diff --git a/chain/deals/handler_utils.go b/chain/deals/provider_utils.go similarity index 66% rename from chain/deals/handler_utils.go rename to chain/deals/provider_utils.go index 099a07e7d..49522d92b 100644 --- a/chain/deals/handler_utils.go +++ b/chain/deals/provider_utils.go @@ -17,8 +17,8 @@ import ( "golang.org/x/xerrors" ) -func (h *Handler) failDeal(id cid.Cid, cerr error) { - if err := h.deals.End(id); err != nil { +func (p *Provider) failDeal(id cid.Cid, cerr error) { + if err := p.deals.End(id); err != nil { log.Warnf("deals.End: %s", err) } @@ -29,16 +29,16 @@ func (h *Handler) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) - err := h.sendSignedResponse(StorageDealResponse{ + err := p.sendSignedResponse(StorageDealResponse{ State: api.DealFailed, Message: cerr.Error(), Proposal: id, }) - s, ok := h.conns[id] + s, ok := p.conns[id] if ok { _ = s.Reset() - delete(h.conns, id) + delete(p.conns, id) } if err != nil { @@ -46,25 +46,29 @@ func (h *Handler) failDeal(id cid.Cid, cerr error) { } } -func (h *Handler) readProposal(s inet.Stream) (proposal SignedStorageDealProposal, err error) { +func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealProposal, err error) { if err := cborrpc.ReadCborRPC(s, &proposal); err != nil { log.Errorw("failed to read proposal message", "error", err) - return SignedStorageDealProposal{}, err + return proposal, err + } + + if err := proposal.Verify(); err != nil { + return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } // TODO: Validate proposal maybe // (and signature, obviously) - if proposal.Proposal.MinerAddress != h.actor { - log.Errorf("proposal with wrong MinerAddress: %s", proposal.Proposal.MinerAddress) - return SignedStorageDealProposal{}, err + if proposal.Provider != p.actor { + log.Errorf("proposal with wrong ProviderAddress: %s", proposal.Provider) + return proposal, err } return } -func (h *Handler) sendSignedResponse(resp StorageDealResponse) error { - s, ok := h.conns[resp.Proposal] +func (p *Provider) sendSignedResponse(resp StorageDealResponse) error { + s, ok := p.conns[resp.Proposal] if !ok { return xerrors.New("couldn't send response: not connected") } @@ -74,12 +78,12 @@ func (h *Handler) sendSignedResponse(resp StorageDealResponse) error { return xerrors.Errorf("serializing response: %w", err) } - worker, err := h.getWorker(h.actor) + worker, err := p.getWorker(p.actor) if err != nil { return err } - sig, err := h.full.WalletSign(context.TODO(), worker, msg) + sig, err := p.full.WalletSign(context.TODO(), worker, msg) if err != nil { return xerrors.Errorf("failed to sign response message: %w", err) } @@ -93,18 +97,18 @@ func (h *Handler) sendSignedResponse(resp StorageDealResponse) error { if err != nil { // Assume client disconnected s.Close() - delete(h.conns, resp.Proposal) + delete(p.conns, resp.Proposal) } return err } -func (h *Handler) getWorker(miner address.Address) (address.Address, error) { +func (p *Provider) getWorker(miner address.Address) (address.Address, error) { getworker := &types.Message{ To: miner, From: miner, Method: actors.MAMethods.GetWorkerAddr, } - r, err := h.full.StateCall(context.TODO(), getworker, nil) + r, err := p.full.StateCall(context.TODO(), getworker, nil) if err != nil { return address.Undef, xerrors.Errorf("getting worker address: %w", err) } diff --git a/chain/deals/types.go b/chain/deals/types.go index 58970700e..b8b38fbf0 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -1,21 +1,16 @@ package deals import ( - "github.com/filecoin-project/lotus/api" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" ) func init() { - cbor.RegisterCborType(StorageDealProposal{}) - cbor.RegisterCborType(SignedStorageDealProposal{}) - - cbor.RegisterCborType(PieceInclusionProof{}) - cbor.RegisterCborType(StorageDealResponse{}) cbor.RegisterCborType(SignedStorageDealResponse{}) @@ -23,60 +18,29 @@ func init() { cbor.RegisterCborType(AskResponse{}) } -const ProtocolID = "/fil/storage/mk/1.0.0" +const DealProtocolID = "/fil/storage/mk/1.0.0" const AskProtocolID = "/fil/storage/ask/1.0.0" -type SerializationMode string - -const ( - SerializationUnixFs = "UnixFs" - SerializationRaw = "Raw" - SerializationIPLD = "IPLD" -) - -type StorageDealProposal struct { - PieceRef cid.Cid // TODO: port to spec - SerializationMode SerializationMode - CommP []byte - - Size uint64 - TotalPrice types.BigInt - Duration uint64 - - Payment actors.PaymentInfo - - MinerAddress address.Address - ClientAddress address.Address -} - -type SignedStorageDealProposal struct { - Proposal StorageDealProposal - - Signature *types.Signature -} - -// response - -type PieceInclusionProof struct { - Position uint64 - ProofElements []byte +type Proposal struct { + DealProposal actors.StorageDealProposal } type StorageDealResponse struct { State api.DealState - // DealRejected / DealAccepted / DealFailed / DealStaged + // DealProposalRejected Message string Proposal cid.Cid - // DealSealing - PieceInclusionProof PieceInclusionProof - CommD []byte // TODO: not in spec + // DealAccepted + StorageDeal actors.StorageDeal + PublishMessage cid.Cid // DealComplete - SectorCommitMessage *cid.Cid + CommitMessage cid.Cid } +// TODO: Do we actually need this to be signed? type SignedStorageDealResponse struct { Response StorageDealResponse diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 06ade0030..12b842473 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -56,7 +56,7 @@ func SetupInitActor(bs bstore.Blockstore, addrs []address.Address) (*types.Actor } act := &types.Actor{ - Code: actors.InitActorCodeCid, + Code: actors.InitCodeCid, Head: statecid, } @@ -85,7 +85,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types return nil, xerrors.Errorf("setup init actor: %w", err) } - if err := state.SetActor(actors.InitActorAddress, initact); err != nil { + if err := state.SetActor(actors.InitAddress, initact); err != nil { return nil, xerrors.Errorf("set init actor: %w", err) } @@ -104,7 +104,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types } err = state.SetActor(actors.NetworkAddress, &types.Actor{ - Code: actors.AccountActorCodeCid, + Code: actors.AccountCodeCid, Balance: netAmt, Head: emptyobject, }) @@ -113,7 +113,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types } err = state.SetActor(actors.BurntFundsAddress, &types.Actor{ - Code: actors.AccountActorCodeCid, + Code: actors.AccountCodeCid, Balance: types.NewInt(0), Head: emptyobject, }) @@ -123,7 +123,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types for a, v := range actmap { err = state.SetActor(a, &types.Actor{ - Code: actors.AccountActorCodeCid, + Code: actors.AccountCodeCid, Balance: v, Head: emptyobject, }) @@ -154,7 +154,7 @@ func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { } return &types.Actor{ - Code: actors.StoragePowerActorCodeCid, + Code: actors.StoragePowerCodeCid, Head: stcid, Nonce: 0, Balance: types.NewInt(0), @@ -333,7 +333,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B } b := &types.BlockHeader{ - Miner: actors.InitActorAddress, + Miner: actors.InitAddress, Tickets: []*types.Ticket{genesisticket}, ElectionProof: []byte("the Genesis block"), Parents: []cid.Cid{}, diff --git a/chain/state/statetree.go b/chain/state/statetree.go index 4f1cdb36d..55a812275 100644 --- a/chain/state/statetree.go +++ b/chain/state/statetree.go @@ -68,7 +68,7 @@ func (st *StateTree) SetActor(addr address.Address, act *types.Actor) error { } func (st *StateTree) lookupID(addr address.Address) (address.Address, error) { - act, err := st.GetActor(actors.InitActorAddress) + act, err := st.GetActor(actors.InitAddress) if err != nil { return address.Undef, xerrors.Errorf("getting init actor: %w", err) } @@ -143,7 +143,7 @@ func (st *StateTree) Snapshot() error { func (st *StateTree) RegisterNewAddress(addr address.Address, act *types.Actor) (address.Address, error) { var out address.Address - err := st.MutateActor(actors.InitActorAddress, func(initact *types.Actor) error { + err := st.MutateActor(actors.InitAddress, func(initact *types.Actor) error { var ias actors.InitActorState if err := st.Store.Get(context.TODO(), initact.Head, &ias); err != nil { return err diff --git a/chain/state/statetree_test.go b/chain/state/statetree_test.go index 79629daeb..360f4778e 100644 --- a/chain/state/statetree_test.go +++ b/chain/state/statetree_test.go @@ -27,7 +27,7 @@ func BenchmarkStateTreeSet(b *testing.B) { err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523), Code: actors.StorageMinerCodeCid, - Head: actors.AccountActorCodeCid, + Head: actors.AccountCodeCid, Nonce: uint64(i), }) if err != nil { @@ -54,7 +54,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) { err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523), Code: actors.StorageMinerCodeCid, - Head: actors.AccountActorCodeCid, + Head: actors.AccountCodeCid, Nonce: uint64(i), }) if err != nil { @@ -80,7 +80,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) { err = st.SetActor(a, &types.Actor{ Balance: types.NewInt(1258812523 + uint64(i)), Code: actors.StorageMinerCodeCid, - Head: actors.AccountActorCodeCid, + Head: actors.AccountCodeCid, Nonce: uint64(i), }) if err != nil { @@ -123,7 +123,7 @@ func TestSetCache(t *testing.T) { act := &types.Actor{ Balance: types.NewInt(0), Code: actors.StorageMinerCodeCid, - Head: actors.AccountActorCodeCid, + Head: actors.AccountCodeCid, Nonce: 0, } diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index c91109734..decc590b7 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -29,12 +29,12 @@ func newInvoker() *invoker { } // add builtInCode using: register(cid, singleton) - inv.register(actors.InitActorCodeCid, actors.InitActor{}, actors.InitActorState{}) - inv.register(actors.StoragePowerActorCodeCid, actors.StoragePowerActor{}, actors.StoragePowerState{}) - inv.register(actors.StorageMarketActorCodeCid, actors.StorageMarketActor{}, actors.StorageMarketState{}) + inv.register(actors.InitCodeCid, actors.InitActor{}, actors.InitActorState{}) + inv.register(actors.StoragePowerCodeCid, actors.StoragePowerActor{}, actors.StoragePowerState{}) + inv.register(actors.StorageMarketCodeCid, actors.StorageMarketActor{}, actors.StorageMarketState{}) inv.register(actors.StorageMinerCodeCid, actors.StorageMinerActor{}, actors.StorageMinerActorState{}) - inv.register(actors.MultisigActorCodeCid, actors.MultiSigActor{}, actors.MultiSigActorState{}) - inv.register(actors.PaymentChannelActorCodeCid, actors.PaymentChannelActor{}, actors.PaymentChannelActorState{}) + inv.register(actors.MultisigCodeCid, actors.MultiSigActor{}, actors.MultiSigActorState{}) + inv.register(actors.PaymentChannelCodeCid, actors.PaymentChannelActor{}, actors.PaymentChannelActorState{}) return inv } diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 6b0fc4ca4..1e1885825 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -66,7 +66,7 @@ func NewBLSAccountActor(st *state.StateTree, addr address.Address) (*types.Actor } nact := &types.Actor{ - Code: actors.AccountActorCodeCid, + Code: actors.AccountCodeCid, Balance: types.NewInt(0), Head: c, } @@ -76,7 +76,7 @@ func NewBLSAccountActor(st *state.StateTree, addr address.Address) (*types.Actor func NewSecp256k1AccountActor(st *state.StateTree, addr address.Address) (*types.Actor, aerrors.ActorError) { nact := &types.Actor{ - Code: actors.AccountActorCodeCid, + Code: actors.AccountCodeCid, Balance: types.NewInt(0), Head: EmptyObjectCid, } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 03faebc29..a37edeb41 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -177,7 +177,7 @@ func (vmc *VMContext) ChargeGas(amount uint64) aerrors.ActorError { } func (vmc *VMContext) StateTree() (types.StateTree, aerrors.ActorError) { - if vmc.msg.To != actors.InitActorAddress { + if vmc.msg.To != actors.InitAddress { return nil, aerrors.Escalate(fmt.Errorf("only init actor can access state tree directly"), "invalid use of StateTree") } @@ -216,7 +216,7 @@ func ResolveToKeyAddr(state types.StateTree, cst *hamt.CborIpldStore, addr addre return address.Undef, aerrors.Newf(1, "failed to find actor: %s", addr) } - if act.Code != actors.AccountActorCodeCid { + if act.Code != actors.AccountCodeCid { return address.Undef, aerrors.New(1, "address was not for an account actor") } diff --git a/node/builder.go b/node/builder.go index 00ab37328..8a59339cc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -256,7 +256,7 @@ func Online() Option { Override(new(dtypes.StagingDAG), modules.StagingDAG), Override(new(*retrieval.Miner), retrieval.NewMiner), - Override(new(*deals.Handler), deals.NewHandler), + Override(new(*deals.Provider), deals.NewProvider), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleDealsKey, modules.HandleDeals), Override(RunSectorServiceKey, modules.RunSectorService), diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 6269fb932..add507cce 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -114,9 +114,9 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A ChannelMessage: payment.ChannelMessage, Vouchers: payment.Vouchers, }, - MinerAddress: miner, - ClientAddress: self, - MinerID: pid, + ProviderAddress: miner, + Client: self, + MinerID: pid, } c, err := a.DealClient.Start(ctx, proposal, vd) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 009ba57b2..54f800a5f 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -98,13 +98,13 @@ func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) { }) } -func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Handler) { +func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Provider) { ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ OnStart: func(context.Context) error { h.Run(ctx) - host.SetStreamHandler(deals.ProtocolID, h.HandleStream) + host.SetStreamHandler(deals.DealProtocolID, h.HandleStream) host.SetStreamHandler(deals.AskProtocolID, h.HandleAskStream) return nil }, diff --git a/paych/simple.go b/paych/simple.go index d8a3f4163..99c578d05 100644 --- a/paych/simple.go +++ b/paych/simple.go @@ -19,14 +19,14 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am enc, aerr := actors.SerializeParams(&actors.ExecParams{ Params: params, - Code: actors.PaymentChannelActorCodeCid, + Code: actors.PaymentChannelCodeCid, }) if aerr != nil { return address.Undef, cid.Undef, aerr } msg := &types.Message{ - To: actors.InitActorAddress, + To: actors.InitAddress, From: from, Value: amt, Method: actors.IAMethods.Exec, From 99ef51a642fcb0453884648d5ff851deb266f020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 12:09:36 +0200 Subject: [PATCH 013/230] on chain deals: Fix build --- api/api.go | 7 +- api/struct.go | 34 ++++--- api/types.go | 6 +- api/utils.go | 1 + chain/actors/actor_storagemarket.go | 31 ++---- chain/actors/cbor_gen.go | 141 ---------------------------- chain/deals/client.go | 4 +- chain/deals/provider_states.go | 14 +-- chain/deals/types.go | 2 +- chain/deals/vouchers.go | 31 ------ node/impl/client/client.go | 47 ++-------- node/impl/full/state.go | 14 +++ 12 files changed, 66 insertions(+), 266 deletions(-) delete mode 100644 chain/deals/vouchers.go diff --git a/api/api.go b/api/api.go index 12e96d69d..696e30d61 100644 --- a/api/api.go +++ b/api/api.go @@ -134,7 +134,7 @@ type FullNode interface { StateWaitMsg(context.Context, cid.Cid) (*MsgWait, error) StateListMiners(context.Context, *types.TipSet) ([]address.Address, error) StateListActors(context.Context, *types.TipSet) ([]address.Address, error) - StateMarketBalance(context.Context, address.Address) (actors.StorageParticipantBalance, error) + StateMarketBalance(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) @@ -199,10 +199,9 @@ type Import struct { type DealInfo struct { ProposalCid cid.Cid State DealState - Miner address.Address + Provider address.Address - PieceRef cid.Cid - CommP []byte + PieceRef []byte // cid bytes Size uint64 TotalPrice types.BigInt diff --git a/api/struct.go b/api/struct.go index b4bd36758..9e1d5b623 100644 --- a/api/struct.go +++ b/api/struct.go @@ -2,6 +2,7 @@ package api import ( "context" + "github.com/filecoin-project/lotus/chain/actors" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" @@ -85,20 +86,21 @@ type FullNodeStruct struct { ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` - StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` - StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` - StateMinerProvingPeriodEnd func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` - StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` - StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"` - StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"` - StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"` - StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` - StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` + StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` + StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` + StateMinerProvingPeriodEnd func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` + StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` + StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"` + StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` + StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"` + StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` + StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"` + StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` + StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` + StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` @@ -388,6 +390,10 @@ func (c *FullNodeStruct) StateListActors(ctx context.Context, ts *types.TipSet) return c.Internal.StateListActors(ctx, ts) } +func (c *FullNodeStruct) StateMarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (actors.StorageParticipantBalance, error) { + return c.Internal.StateMarketBalance(ctx, addr, ts) +} + func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } diff --git a/api/types.go b/api/types.go index f584345f3..694264f5d 100644 --- a/api/types.go +++ b/api/types.go @@ -9,11 +9,11 @@ import ( type DealState int const ( - DealUnknown = DealState(iota) + DealUnknown = DealState(iota) DealRejected // Provider didn't like the proposal DealAccepted // Proposal accepted, data moved - DealStaged // Data put into the sector - DealSealing // Data in process of being sealed + DealStaged // Data put into the sector + DealSealing // Data in process of being sealed DealFailed diff --git a/api/utils.go b/api/utils.go index 5bbac426e..9927d44d0 100644 --- a/api/utils.go +++ b/api/utils.go @@ -24,4 +24,5 @@ func SignWith(ctx context.Context, signer Signer, addr address.Address, signable return err } } + return nil } diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 3753621d4..abac85542 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -143,7 +143,7 @@ func (sma StorageMarketActor) WithdrawBalance(act *types.Actor, vmctx types.VMCo return nil, err } - b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, vmctx.Message().From) if err != nil { return nil, aerrors.Wrap(err, "could not get balance") } @@ -185,7 +185,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext return nil, err } - b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{vmctx.Message().From}) + b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, vmctx.Message().From) if err != nil { return nil, aerrors.Wrap(err, "could not get balance") } @@ -228,8 +228,8 @@ func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Add return c, nil } -func getMarketBalances(vmctx types.VMContext, rcid cid.Cid, addrs []address.Address) ([]StorageParticipantBalance, *hamt.Node, ActorError) { - nd, err := hamt.LoadNode(vmctx.Context(), vmctx.Ipld(), rcid) +func GetMarketBalances(ctx context.Context, store *hamt.CborIpldStore, rcid cid.Cid, addrs ...address.Address) ([]StorageParticipantBalance, *hamt.Node, ActorError) { + nd, err := hamt.LoadNode(ctx, store, rcid) if err != nil { return nil, nil, aerrors.HandleExternalError(err, "failed to load miner set") } @@ -238,7 +238,7 @@ func getMarketBalances(vmctx types.VMContext, rcid cid.Cid, addrs []address.Addr for i, a := range addrs { var balance StorageParticipantBalance - err = nd.Find(vmctx.Context(), string(a.Bytes()), &balance) + err = nd.Find(ctx, string(a.Bytes()), &balance) switch err { case hamt.ErrNotFound: out[i] = StorageParticipantBalance{ @@ -336,12 +336,6 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage if vmctx.BlockHeight() >= deal.Proposal.ProposalExpiration { return aerrors.New(1, "deal proposal already expired") } - if vmctx.BlockHeight() >= deal.Proposal.DealExpiration { - return aerrors.New(2, "deal proposal already expired") - } - if deal.Proposal.ProposalExpiration > deal.Proposal.DealExpiration { - return aerrors.New(3, "ProposalExpiration > DealExpiration") - } var proposalBuf bytes.Buffer err := deal.Proposal.MarshalCBOR(&proposalBuf) @@ -373,10 +367,7 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage // TODO: REVIEW: Do we want to check if provider exists in the power actor? // TODO: do some caching (changes gas so needs to be in spec too) - b, bnd, aerr := getMarketBalances(vmctx, self.Balances, []address.Address{ - deal.Proposal.Client, - deal.Proposal.Provider, - }) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, deal.Proposal.Client, deal.Proposal.Provider) if aerr != nil { return aerrors.Wrap(aerr, "getting client, and provider balances") } @@ -518,17 +509,11 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, nil } - // TODO: clients probably want this to be fixed - dealDuration := dealInfo.Deal.Proposal.DealExpiration - dealInfo.ActivationEpoch - // todo: check math (written on a plane, also tired) // TODO: division is hard, this more than likely has some off-by-one issue - toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealDuration)) + toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealInfo.Deal.Proposal.Duration)) - b, bnd, err := getMarketBalances(vmctx, self.Balances, []address.Address{ - dealInfo.Deal.Proposal.Client, - dealInfo.Deal.Proposal.Provider, - }) + b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, dealInfo.Deal.Proposal.Provider) clientBal := b[0] providerBal := b[1] diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 85a199270..6431cc439 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -792,143 +792,6 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *PieceInclVoucherData) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.t.CommP ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil { - return err - } - if _, err := w.Write(t.CommP); err != nil { - return err - } - - // t.t.PieceSize (types.BigInt) - if err := t.PieceSize.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *PieceInclVoucherData) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.t.CommP ([]uint8) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.CommP: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommP = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommP); err != nil { - return err - } - // t.t.PieceSize (types.BigInt) - - { - - if err := t.PieceSize.UnmarshalCBOR(br); err != nil { - return err - } - - } - return nil -} - -func (t *InclusionProof) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.t.Sector (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Sector)); err != nil { - return err - } - - // t.t.Proof ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { - return err - } - if _, err := w.Write(t.Proof); err != nil { - return err - } - return nil -} - -func (t *InclusionProof) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.t.Sector (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Sector = extra - // t.t.Proof ([]uint8) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Proof = make([]byte, extra) - if _, err := io.ReadFull(br, t.Proof); err != nil { - return err - } - return nil -} - func (t *PaymentVerifyParams) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -3087,9 +2950,6 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { } // t.t.DealExpiration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DealExpiration)); err != nil { - return err - } // t.t.StoragePrice (types.BigInt) if err := t.StoragePrice.MarshalCBOR(w); err != nil { @@ -3187,7 +3047,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.DealExpiration = extra // t.t.StoragePrice (types.BigInt) { diff --git a/chain/deals/client.go b/chain/deals/client.go index 8e889836d..614a2e6ff 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -46,7 +46,7 @@ type ClientDeal struct { type Client struct { sm *stmgr.StateManager - chain *store.ChainStore + chain *store.ChainStore h host.Host w *wallet.Wallet dag dtypes.ClientDAG @@ -71,7 +71,7 @@ type clientDealUpdate struct { func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { c := &Client{ sm: sm, - chain: chain, + chain: chain, h: h, w: w, dag: dag, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index a7c321faa..0d3e116ca 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -78,7 +78,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral / StoragePrice // check market funds - clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client) + clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.New("clientMarketBalance.Available too small") } - providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client) + providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) if err != nil { return nil, err } @@ -125,9 +125,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) log.Info("fetching data for a deal") err = p.sendSignedResponse(StorageDealResponse{ - State: api.DealAccepted, - Message: "", - Proposal: deal.ProposalCid, + State: api.DealAccepted, + Message: "", + Proposal: deal.ProposalCid, PublishMessage: smsg.Cid(), }) if err != nil { @@ -206,8 +206,8 @@ func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilde func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { err := p.sendSignedResponse(StorageDealResponse{ - State: api.DealSealing, - Proposal: deal.ProposalCid, + State: api.DealSealing, + Proposal: deal.ProposalCid, }) if err != nil { log.Warnf("Sending deal response failed: %s", err) diff --git a/chain/deals/types.go b/chain/deals/types.go index b8b38fbf0..54bac241a 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -33,7 +33,7 @@ type StorageDealResponse struct { Proposal cid.Cid // DealAccepted - StorageDeal actors.StorageDeal + StorageDeal actors.StorageDeal PublishMessage cid.Cid // DealComplete diff --git a/chain/deals/vouchers.go b/chain/deals/vouchers.go deleted file mode 100644 index f854299b0..000000000 --- a/chain/deals/vouchers.go +++ /dev/null @@ -1,31 +0,0 @@ -package deals - -import ( - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" -) - -func VoucherSpec(blocksDuration uint64, price types.BigInt, start uint64, extra *types.ModVerifyParams) []api.VoucherSpec { - nVouchers := blocksDuration / build.MinDealVoucherIncrement - if nVouchers < 1 { - nVouchers = 1 - } - if nVouchers > build.MaxVouchersPerDeal { - nVouchers = build.MaxVouchersPerDeal - } - - hIncrements := blocksDuration / nVouchers - vouchers := make([]api.VoucherSpec, nVouchers) - - for i := uint64(0); i < nVouchers; i++ { - vouchers[i] = api.VoucherSpec{ - Amount: types.BigDiv(types.BigMul(price, types.NewInt(i+1)), types.NewInt(nVouchers)), - TimeLock: start + (hIncrements * (i + 1)), - MinClose: start + (hIncrements * (i + 1)), - Extra: extra, - } - } - - return vouchers -} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index add507cce..a5c1ecb81 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -13,7 +13,6 @@ import ( chunker "github.com/ipfs/go-ipfs-chunker" offline "github.com/ipfs/go-ipfs-exchange-offline" files "github.com/ipfs/go-ipfs-files" - cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" @@ -76,50 +75,19 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A return nil, err } - vd, err := a.DealClient.VerifyParams(ctx, data) - if err != nil { - return nil, err - } - - voucherData, err := cbor.DumpObject(vd) - if err != nil { - return nil, err - } - // setup payments total := types.BigMul(price, types.NewInt(blocksDuration)) - // TODO: at least ping the miner before creating paych / locking the money - extra := &types.ModVerifyParams{ - Actor: miner, - Method: actors.MAMethods.PaymentVerifyInclusion, - Data: voucherData, - } - - head := a.Chain.GetHeaviestTipSet() - vouchers := deals.VoucherSpec(blocksDuration, total, head.Height(), extra) - - payment, err := a.PaychNewPayment(ctx, self, miner, vouchers) - if err != nil { - return nil, err - } - proposal := deals.ClientDealProposal{ - Data: data, - TotalPrice: total, - Duration: blocksDuration, - Payment: actors.PaymentInfo{ - PayChActor: payment.Channel, - Payer: self, - ChannelMessage: payment.ChannelMessage, - Vouchers: payment.Vouchers, - }, + Data: data, + TotalPrice: total, + Duration: blocksDuration, ProviderAddress: miner, Client: self, MinerID: pid, } - c, err := a.DealClient.Start(ctx, proposal, vd) + c, err := a.DealClient.Start(ctx, proposal) // TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!) return &c, err } @@ -135,13 +103,12 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { out[k] = api.DealInfo{ ProposalCid: v.ProposalCid, State: v.State, - Miner: v.Proposal.MinerAddress, + Provider: v.Proposal.Provider, PieceRef: v.Proposal.PieceRef, - CommP: v.Proposal.CommP, - Size: v.Proposal.Size, + Size: v.Proposal.PieceSize, - TotalPrice: v.Proposal.TotalPrice, + TotalPrice: v.Proposal.StoragePrice, Duration: v.Proposal.Duration, } } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e357c0814..c6a8e2e15 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -226,3 +226,17 @@ func (a *StateAPI) StateListMiners(ctx context.Context, ts *types.TipSet) ([]add func (a *StateAPI) StateListActors(ctx context.Context, ts *types.TipSet) ([]address.Address, error) { return a.StateManager.ListAllActors(ctx, ts) } + +func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (actors.StorageParticipantBalance, error) { + var state actors.StorageMarketState + if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + return actors.StorageParticipantBalance{}, err + } + cst := hamt.CSTFromBstore(a.StateManager.ChainStore().Blockstore()) + b, _, err := actors.GetMarketBalances(ctx, cst, state.Balances, addr) + if err != nil { + return actors.StorageParticipantBalance{}, err + } + + return b[0], nil +} From bafb7e8cfa246ed4321df0d07b1f0be339ec27f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 12:20:43 +0200 Subject: [PATCH 014/230] on chain deals: deal types cbor-gen --- api/types.go | 2 +- chain/actors/actor_storagemarket.go | 2 +- chain/actors/cbor_gen.go | 41 +- chain/deals/cbor_gen.go | 730 ++++++++++++++++++++++++++++ chain/deals/client.go | 7 - chain/deals/client_utils.go | 6 +- chain/deals/provider.go | 4 - chain/deals/provider_asks.go | 4 +- chain/deals/provider_states.go | 8 +- chain/deals/provider_utils.go | 6 +- chain/deals/types.go | 18 +- chain/types/ask.go | 4 +- chain/types/cbor_gen.go | 198 ++++++++ gen/main.go | 25 +- 14 files changed, 1005 insertions(+), 50 deletions(-) create mode 100644 chain/deals/cbor_gen.go diff --git a/api/types.go b/api/types.go index 694264f5d..475eace60 100644 --- a/api/types.go +++ b/api/types.go @@ -6,7 +6,7 @@ import ( ma "github.com/multiformats/go-multiaddr" ) -type DealState int +type DealState = uint64 const ( DealUnknown = DealState(iota) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index abac85542..36e5edc9e 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -61,7 +61,7 @@ type StorageMarketState struct { } // TODO: serialization mode spec -type SerializationMode uint64 +type SerializationMode = uint64 const ( SerializationUnixFSv0 = iota diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 6431cc439..cc580f47f 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -2917,7 +2917,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{137}); err != nil { + if _, err := w.Write([]byte{138}); err != nil { return err } @@ -2934,6 +2934,11 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } + // t.t.PieceSerialization (actors.SerializationMode) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSerialization)); err != nil { + return err + } + // t.t.Client (address.Address) if err := t.Client.MarshalCBOR(w); err != nil { return err @@ -2949,7 +2954,10 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealExpiration (uint64) + // t.t.Duration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + return err + } // t.t.StoragePrice (types.BigInt) if err := t.StoragePrice.MarshalCBOR(w); err != nil { @@ -2979,7 +2987,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 9 { + if extra != 10 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -3010,6 +3018,16 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.PieceSize = extra + // t.t.PieceSerialization (actors.SerializationMode) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PieceSerialization = extra // t.t.Client (address.Address) { @@ -3038,7 +3056,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.ProposalExpiration = extra - // t.t.DealExpiration (uint64) + // t.t.Duration (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3047,6 +3065,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } + t.Duration = extra // t.t.StoragePrice (types.BigInt) { @@ -3069,9 +3088,21 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { { - if err := t.ProposerSignature.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.ProposerSignature = new(types.Signature) + if err := t.ProposerSignature.UnmarshalCBOR(br); err != nil { + return err + } + } } return nil diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go new file mode 100644 index 000000000..9952630e3 --- /dev/null +++ b/chain/deals/cbor_gen.go @@ -0,0 +1,730 @@ +package deals + +import ( + "fmt" + "io" + + "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/types" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *AskRequest) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Miner (address.Address) + if err := t.Miner.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Miner (address.Address) + + { + + if err := t.Miner.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *AskResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Ask (types.SignedStorageAsk) + if err := t.Ask.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Ask (types.SignedStorageAsk) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Ask = new(types.SignedStorageAsk) + if err := t.Ask.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + return nil +} + +func (t *Proposal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.DealProposal (actors.StorageDealProposal) + if err := t.DealProposal.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *Proposal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealProposal (actors.StorageDealProposal) + + { + + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *Response) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{134}); err != nil { + return err + } + + // t.t.State (api.DealState) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + return err + } + + // t.t.Message (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Message)); err != nil { + return err + } + + // t.t.Proposal (cid.Cid) + + if err := cbg.WriteCid(w, t.Proposal); err != nil { + return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) + } + + // t.t.StorageDeal (actors.StorageDeal) + if err := t.StorageDeal.MarshalCBOR(w); err != nil { + return err + } + + // t.t.PublishMessage (cid.Cid) + + if err := cbg.WriteCid(w, t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } + + // t.t.CommitMessage (cid.Cid) + + if err := cbg.WriteCid(w, t.CommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + } + + return nil +} + +func (t *Response) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.State (api.DealState) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = extra + // t.t.Message (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Message = string(sval) + } + // t.t.Proposal (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Proposal: %w", err) + } + + t.Proposal = c + + } + // t.t.StorageDeal (actors.StorageDeal) + + { + + if err := t.StorageDeal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.PublishMessage (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + } + + t.PublishMessage = c + + } + // t.t.CommitMessage (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = c + + } + return nil +} + +func (t *SignedResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Response (deals.Response) + if err := t.Response.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Signature (types.Signature) + if err := t.Signature.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Response (deals.Response) + + { + + if err := t.Response.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Signature (types.Signature) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Signature = new(types.Signature) + if err := t.Signature.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + return nil +} + +func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{136}); err != nil { + return err + } + + // t.t.Data (cid.Cid) + + if err := cbg.WriteCid(w, t.Data); err != nil { + return xerrors.Errorf("failed to write cid field t.Data: %w", err) + } + + // t.t.DataSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DataSize)); err != nil { + return err + } + + // t.t.TotalPrice (types.BigInt) + if err := t.TotalPrice.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ProposalExpiration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + return err + } + + // t.t.Duration (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + return err + } + + // t.t.ProviderAddress (address.Address) + if err := t.ProviderAddress.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Client (address.Address) + if err := t.Client.MarshalCBOR(w); err != nil { + return err + } + + // t.t.MinerID (peer.ID) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.MinerID)); err != nil { + return err + } + return nil +} + +func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 8 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Data (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Data: %w", err) + } + + t.Data = c + + } + // t.t.DataSize (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DataSize = extra + // t.t.TotalPrice (types.BigInt) + + { + + if err := t.TotalPrice.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ProposalExpiration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ProposalExpiration = extra + // t.t.Duration (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Duration = extra + // t.t.ProviderAddress (address.Address) + + { + + if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Client (address.Address) + + { + + if err := t.Client.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.MinerID (peer.ID) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.MinerID = peer.ID(sval) + } + return nil +} + +func (t *ClientDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{132}); err != nil { + return err + } + + // t.t.ProposalCid (cid.Cid) + + if err := cbg.WriteCid(w, t.ProposalCid); err != nil { + return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) + } + + // t.t.Proposal (actors.StorageDealProposal) + if err := t.Proposal.MarshalCBOR(w); err != nil { + return err + } + + // t.t.State (api.DealState) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + return err + } + + // t.t.Miner (peer.ID) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Miner)); err != nil { + return err + } + return nil +} + +func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.ProposalCid (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + } + + t.ProposalCid = c + + } + // t.t.Proposal (actors.StorageDealProposal) + + { + + if err := t.Proposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.State (api.DealState) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = extra + // t.t.Miner (peer.ID) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Miner = peer.ID(sval) + } + return nil +} + +func (t *MinerDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{134}); err != nil { + return err + } + + // t.t.Client (peer.ID) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Client)); err != nil { + return err + } + + // t.t.Proposal (actors.StorageDealProposal) + if err := t.Proposal.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ProposalCid (cid.Cid) + + if err := cbg.WriteCid(w, t.ProposalCid); err != nil { + return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) + } + + // t.t.State (api.DealState) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + return err + } + + // t.t.Ref (cid.Cid) + + if err := cbg.WriteCid(w, t.Ref); err != nil { + return xerrors.Errorf("failed to write cid field t.Ref: %w", err) + } + + // t.t.SectorID (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + return err + } + return nil +} + +func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Client (peer.ID) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Client = peer.ID(sval) + } + // t.t.Proposal (actors.StorageDealProposal) + + { + + if err := t.Proposal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ProposalCid (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err) + } + + t.ProposalCid = c + + } + // t.t.State (api.DealState) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = extra + // t.t.Ref (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Ref: %w", err) + } + + t.Ref = c + + } + // t.t.SectorID (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = extra + return nil +} diff --git a/chain/deals/client.go b/chain/deals/client.go index 614a2e6ff..3df2e5018 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -26,13 +26,6 @@ import ( "github.com/filecoin-project/lotus/retrieval/discovery" ) -func init() { - cbor.RegisterCborType(ClientDeal{}) - cbor.RegisterCborType(types.SignedVoucher{}) - cbor.RegisterCborType(types.ModVerifyParams{}) - cbor.RegisterCborType(types.Signature{}) -} - var log = logging.Logger("deals") type ClientDeal struct { diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 648e41733..eb05cb8ca 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -25,16 +25,16 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) } -func (c *Client) readStorageDealResp(deal ClientDeal) (*StorageDealResponse, error) { +func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { s, ok := c.conns[deal.ProposalCid] if !ok { // TODO: Try to re-establish the connection using query protocol return nil, xerrors.Errorf("no connection to miner") } - var resp SignedStorageDealResponse + var resp SignedResponse if err := cborrpc.ReadCborRPC(s, &resp); err != nil { - log.Errorw("failed to read StorageDealResponse message", "error", err) + log.Errorw("failed to read Response message", "error", err) return nil, err } diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 67fbf33f1..22d94c36b 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -22,10 +22,6 @@ import ( "github.com/filecoin-project/lotus/storage/sectorblocks" ) -func init() { - cbor.RegisterCborType(MinerDeal{}) -} - type MinerDeal struct { Client peer.ID Proposal actors.StorageDealProposal diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index af7c8cce6..27faeae19 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -26,8 +26,8 @@ func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error { now := time.Now().Unix() ask := &types.StorageAsk{ Price: price, - Timestamp: now, - Expiry: now + ttlsecs, + Timestamp: uint64(now), + Expiry: uint64(now + ttlsecs), Miner: p.actor, SeqNo: seqno, MinPieceSize: p.minPieceSize, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 0d3e116ca..ed7baea86 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -124,7 +124,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } log.Info("fetching data for a deal") - err = p.sendSignedResponse(StorageDealResponse{ + err = p.sendSignedResponse(Response{ State: api.DealAccepted, Message: "", Proposal: deal.ProposalCid, @@ -140,7 +140,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // STAGED func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(StorageDealResponse{ + err := p.sendSignedResponse(Response{ State: api.DealStaged, Proposal: deal.ProposalCid, }) @@ -205,7 +205,7 @@ func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilde } func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(StorageDealResponse{ + err := p.sendSignedResponse(Response{ State: api.DealSealing, Proposal: deal.ProposalCid, }) @@ -231,7 +231,7 @@ func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDea log.Warnf("Waiting for sector commitment message: %s", err) } - err = p.sendSignedResponse(StorageDealResponse{ + err = p.sendSignedResponse(Response{ State: api.DealComplete, Proposal: deal.ProposalCid, diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 49522d92b..3a9f96233 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -29,7 +29,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) - err := p.sendSignedResponse(StorageDealResponse{ + err := p.sendSignedResponse(Response{ State: api.DealFailed, Message: cerr.Error(), Proposal: id, @@ -67,7 +67,7 @@ func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealPropo return } -func (p *Provider) sendSignedResponse(resp StorageDealResponse) error { +func (p *Provider) sendSignedResponse(resp Response) error { s, ok := p.conns[resp.Proposal] if !ok { return xerrors.New("couldn't send response: not connected") @@ -88,7 +88,7 @@ func (p *Provider) sendSignedResponse(resp StorageDealResponse) error { return xerrors.Errorf("failed to sign response message: %w", err) } - signedResponse := SignedStorageDealResponse{ + signedResponse := SignedResponse{ Response: resp, Signature: sig, } diff --git a/chain/deals/types.go b/chain/deals/types.go index 54bac241a..d4b445328 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -1,23 +1,13 @@ package deals import ( - "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" ) -func init() { - cbor.RegisterCborType(StorageDealResponse{}) - cbor.RegisterCborType(SignedStorageDealResponse{}) - - cbor.RegisterCborType(AskRequest{}) - cbor.RegisterCborType(AskResponse{}) -} - const DealProtocolID = "/fil/storage/mk/1.0.0" const AskProtocolID = "/fil/storage/ask/1.0.0" @@ -25,7 +15,7 @@ type Proposal struct { DealProposal actors.StorageDealProposal } -type StorageDealResponse struct { +type Response struct { State api.DealState // DealProposalRejected @@ -41,8 +31,8 @@ type StorageDealResponse struct { } // TODO: Do we actually need this to be signed? -type SignedStorageDealResponse struct { - Response StorageDealResponse +type SignedResponse struct { + Response Response Signature *types.Signature } diff --git a/chain/types/ask.go b/chain/types/ask.go index da0684f86..1e74c9cb6 100644 --- a/chain/types/ask.go +++ b/chain/types/ask.go @@ -19,7 +19,7 @@ type StorageAsk struct { Price BigInt MinPieceSize uint64 Miner address.Address - Timestamp int64 - Expiry int64 + Timestamp uint64 + Expiry uint64 SeqNo uint64 } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index bad374f1b..9cf6ec913 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1272,3 +1272,201 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { return nil } + +func (t *SignedStorageAsk) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Ask (types.StorageAsk) + if err := t.Ask.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Signature (types.Signature) + if err := t.Signature.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Ask (types.StorageAsk) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Ask = new(StorageAsk) + if err := t.Ask.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.Signature (types.Signature) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Signature = new(Signature) + if err := t.Signature.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + return nil +} + +func (t *StorageAsk) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{134}); err != nil { + return err + } + + // t.t.Price (types.BigInt) + if err := t.Price.MarshalCBOR(w); err != nil { + return err + } + + // t.t.MinPieceSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinPieceSize)); err != nil { + return err + } + + // t.t.Miner (address.Address) + if err := t.Miner.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Timestamp (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Timestamp)); err != nil { + return err + } + + // t.t.Expiry (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Expiry)); err != nil { + return err + } + + // t.t.SeqNo (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SeqNo)); err != nil { + return err + } + return nil +} + +func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Price (types.BigInt) + + { + + if err := t.Price.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.MinPieceSize (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.MinPieceSize = extra + // t.t.Miner (address.Address) + + { + + if err := t.Miner.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Timestamp (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Timestamp = extra + // t.t.Expiry (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Expiry = extra + // t.t.SeqNo (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SeqNo = extra + return nil +} diff --git a/gen/main.go b/gen/main.go index dd5842ed0..5bf4280a4 100644 --- a/gen/main.go +++ b/gen/main.go @@ -4,9 +4,11 @@ import ( "fmt" "os" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" gen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/deals" + "github.com/filecoin-project/lotus/chain/types" ) func main() { @@ -22,6 +24,8 @@ func main() { types.Actor{}, types.MessageReceipt{}, types.BlockMsg{}, + types.SignedStorageAsk{}, + types.StorageAsk{}, ) if err != nil { fmt.Println(err) @@ -49,8 +53,6 @@ func main() { actors.CommitSectorParams{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, - actors.PieceInclVoucherData{}, - actors.InclusionProof{}, actors.PaymentVerifyParams{}, actors.UpdatePeerIDParams{}, actors.MultiSigActorState{}, @@ -89,4 +91,19 @@ func main() { fmt.Println(err) os.Exit(1) } + + err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals", + deals.AskRequest{}, + deals.AskResponse{}, + deals.Proposal{}, + deals.Response{}, + deals.SignedResponse{}, + deals.ClientDealProposal{}, + deals.ClientDeal{}, + deals.MinerDeal{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } From 47d92d4a11f4844597e6591b2376eeea606ce012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 13:29:41 +0200 Subject: [PATCH 015/230] wip --- api/api.go | 2 +- api/struct.go | 2 +- chain/actors/actor_miner.go | 2 +- chain/actors/actor_storagemarket.go | 1 + chain/deals/client.go | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/api.go b/api/api.go index 696e30d61..ea444a2c9 100644 --- a/api/api.go +++ b/api/api.go @@ -3,7 +3,6 @@ package api import ( "context" "fmt" - "github.com/filecoin-project/lotus/chain/actors" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" @@ -14,6 +13,7 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" diff --git a/api/struct.go b/api/struct.go index 9e1d5b623..1f09733f9 100644 --- a/api/struct.go +++ b/api/struct.go @@ -2,13 +2,13 @@ package api import ( "context" - "github.com/filecoin-project/lotus/chain/actors" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 7f87a2aea..b57d33f62 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -210,7 +210,7 @@ type CommitSectorParams struct { } type OnChainSealVerifyInfo struct { - SealedCID cid.Cid // CommR .. TODO: spec says cid, but it feela weird + SealedCID cid.Cid // CommR .. TODO: spec says cid, but it feels weird Epoch uint64 Proof []byte DealIDs []uint64 diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 36e5edc9e..0007fb0b3 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -3,6 +3,7 @@ package actors import ( "bytes" "context" + "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" diff --git a/chain/deals/client.go b/chain/deals/client.go index 3df2e5018..94cb1b8ba 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,7 +2,6 @@ package deals import ( "context" - "github.com/filecoin-project/lotus/chain/store" "math" "github.com/ipfs/go-cid" @@ -19,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/cborrpc" From 64bfb3883416a686d9fe2eba3b049144df8cd8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 19:34:59 +0200 Subject: [PATCH 016/230] on chain deals: Get things to actually run! --- build/params.go | 4 +- chain/actors/actor_init.go | 2 +- chain/actors/actor_miner.go | 34 ++++++++------- chain/actors/cbor_gen.go | 82 +++++++++++++++++++++++++++--------- chain/deals/provider_asks.go | 10 ++--- chain/gen/utils.go | 47 ++++++++++++++++++++- gen/main.go | 2 +- lib/cborrpc/rpc.go | 9 ++++ storage/miner.go | 12 +++--- 9 files changed, 148 insertions(+), 54 deletions(-) diff --git a/build/params.go b/build/params.go index 4f27abe60..780f7d931 100644 --- a/build/params.go +++ b/build/params.go @@ -30,7 +30,7 @@ const MaxVouchersPerDeal = 768 // roughly one voucher per 10h over a year // Consensus / Network // Seconds -const BlockDelay = 30 +const BlockDelay = 3 // Seconds const AllowableClockDrift = BlockDelay * 2 @@ -51,7 +51,7 @@ const RandomnessLookback = 20 const ProvingPeriodDuration = 40 // Blocks -const PoSTChallangeTime = 20 +const PoSTChallangeTime = 35 const PowerCollateralProportion = 5 const PerCapitaCollateralProportion = 1 diff --git a/chain/actors/actor_init.go b/chain/actors/actor_init.go index 3ab1243a8..71aefda70 100644 --- a/chain/actors/actor_init.go +++ b/chain/actors/actor_init.go @@ -170,7 +170,7 @@ func IsBuiltinActor(code cid.Cid) bool { } func IsSingletonActor(code cid.Cid) bool { - return code == StoragePowerCodeCid || code == InitCodeCid + return code == StoragePowerCodeCid || code == StorageMarketCodeCid || code == InitCodeCid } func (ias *InitActorState) AddActor(cst *hamt.CborIpldStore, addr address.Address) (address.Address, error) { diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index b57d33f62..fba8f94ad 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -18,8 +18,6 @@ import ( "golang.org/x/xerrors" ) -const POST_SECTORS_COUNT = 8192 - type StorageMinerActor struct{} type StorageMinerActorState struct { @@ -201,23 +199,18 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ return nil, nil } -type CommitSectorParams struct { - SectorID uint64 - CommD []byte +type OnChainSealVerifyInfo struct { + CommD []byte // TODO: update proofs code CommR []byte CommRStar []byte - Proof []byte -} -type OnChainSealVerifyInfo struct { - SealedCID cid.Cid // CommR .. TODO: spec says cid, but it feels weird - Epoch uint64 + //Epoch uint64 Proof []byte DealIDs []uint64 SectorNumber uint64 } -func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *CommitSectorParams) ([]byte, ActorError) { +func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *OnChainSealVerifyInfo) ([]byte, ActorError) { ctx := context.TODO() oldstate, self, err := loadState(vmctx) if err != nil { @@ -239,7 +232,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex } // make sure the miner isnt trying to submit a pre-existing sector - unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorID) + unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber) if err != nil { return nil, err } @@ -251,6 +244,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex futurePower := types.BigAdd(self.Power, mi.SectorSize) collateralRequired := CollateralForPower(futurePower) + // TODO: grab from market? if act.Balance.LessThan(collateralRequired) { return nil, aerrors.New(3, "not enough collateral") } @@ -258,7 +252,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex // Note: There must exist a unique index in the miner's sector set for each // sector ID. The `faults`, `recovered`, and `done` parameters of the // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, params.CommR, params.CommD) + nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber, params.CommR, params.CommD) if err != nil { return nil, err } @@ -290,7 +284,15 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, err } - return nil, nil + activateParams, err := SerializeParams(&ActivateStorageDealsParams{ + Deals: params.DealIDs, + }) + if err != nil { + return nil, err + } + + _, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams) + return nil, err } type SubmitPoStParams struct { @@ -515,8 +517,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *CommitSectorParams) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorID, params.Proof) +func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *OnChainSealVerifyInfo) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index cc580f47f..f26978564 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -491,17 +491,12 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *CommitSectorParams) MarshalCBOR(w io.Writer) error { +func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{133}); err != nil { - return err - } - - // t.t.SectorID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + if _, err := w.Write([]byte{134}); err != nil { return err } @@ -536,10 +531,25 @@ func (t *CommitSectorParams) MarshalCBOR(w io.Writer) error { if _, err := w.Write(t.Proof); err != nil { return err } + + // t.t.DealIDs ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + + // t.t.SectorNumber (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorNumber)); err != nil { + return err + } return nil } -func (t *CommitSectorParams) UnmarshalCBOR(r io.Reader) error { +func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -550,20 +560,10 @@ func (t *CommitSectorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.SectorID (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = extra // t.t.CommD ([]uint8) maj, extra, err = cbg.CborReadHeader(br) @@ -632,6 +632,46 @@ func (t *CommitSectorParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Proof); err != nil { return err } + // t.t.DealIDs ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + // t.t.SectorNumber (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = extra return nil } @@ -2934,7 +2974,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceSerialization (actors.SerializationMode) + // t.t.PieceSerialization (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSerialization)); err != nil { return err } @@ -3018,7 +3058,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.PieceSize = extra - // t.t.PieceSerialization (actors.SerializationMode) + // t.t.PieceSerialization (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index 27faeae19..ff1f7f7a1 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -1,6 +1,7 @@ package deals import ( + "bytes" "context" "time" @@ -9,7 +10,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborrpc" datastore "github.com/ipfs/go-datastore" - cbor "github.com/ipfs/go-ipld-cbor" inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" ) @@ -98,7 +98,7 @@ func (p *Provider) loadAsk() error { } var ssa types.SignedStorageAsk - if err := cbor.DecodeInto(askb, &ssa); err != nil { + if err := cborrpc.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { return err } @@ -107,7 +107,7 @@ func (p *Provider) loadAsk() error { } func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cbor.DumpObject(a) + b, err := cborrpc.Dump(a) if err != nil { return nil, err } @@ -129,7 +129,7 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) } func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cbor.DumpObject(a) + b, err := cborrpc.Dump(a) if err != nil { return err } @@ -150,7 +150,7 @@ func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { return xerrors.Errorf("failed to get worker for miner in ask", err) } - sigb, err := cbor.DumpObject(ask.Ask) + sigb, err := cborrpc.Dump(ask.Ask) if err != nil { return xerrors.Errorf("failed to re-serialize ask") } diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 12b842473..666b9569c 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -89,12 +89,21 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types return nil, xerrors.Errorf("set init actor: %w", err) } + spact, err := SetupStoragePowerActor(bs) + if err != nil { + return nil, xerrors.Errorf("setup storage market actor: %w", err) + } + + if err := state.SetActor(actors.StoragePowerAddress, spact); err != nil { + return nil, xerrors.Errorf("set storage market actor: %w", err) + } + smact, err := SetupStorageMarketActor(bs) if err != nil { return nil, xerrors.Errorf("setup storage market actor: %w", err) } - if err := state.SetActor(actors.StoragePowerAddress, smact); err != nil { + if err := state.SetActor(actors.StorageMarketAddress, smact); err != nil { return nil, xerrors.Errorf("set storage market actor: %w", err) } @@ -135,7 +144,7 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types return state, nil } -func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { +func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { cst := hamt.CSTFromBstore(bs) nd := hamt.NewNode(cst) emptyhamt, err := cst.Put(context.TODO(), nd) @@ -161,6 +170,40 @@ func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { }, nil } +func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) { + cst := hamt.CSTFromBstore(bs) + nd := hamt.NewNode(cst) + emptyHAMT, err := cst.Put(context.TODO(), nd) + if err != nil { + return nil, err + } + + blks := amt.WrapBlockstore(bs) + + emptyAMT, err := amt.FromArray(blks, nil) + if err != nil { + return nil, xerrors.Errorf("amt build failed: %w", err) + } + + sms := &actors.StorageMarketState{ + Balances: emptyHAMT, + Deals: emptyAMT, + NextDealID: 0, + } + + stcid, err := cst.Put(context.TODO(), sms) + if err != nil { + return nil, err + } + + return &types.Actor{ + Code: actors.StorageMarketCodeCid, + Head: stcid, + Nonce: 0, + Balance: types.NewInt(0), + }, nil +} + type GenMinerCfg struct { Owners []address.Address Workers []address.Address diff --git a/gen/main.go b/gen/main.go index 5bf4280a4..64eae89ab 100644 --- a/gen/main.go +++ b/gen/main.go @@ -50,7 +50,7 @@ func main() { actors.AccountActorState{}, actors.StorageMinerActorState{}, actors.StorageMinerConstructorParams{}, - actors.CommitSectorParams{}, + actors.OnChainSealVerifyInfo{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, actors.PaymentVerifyParams{}, diff --git a/lib/cborrpc/rpc.go b/lib/cborrpc/rpc.go index c02fc5298..98a5e71ec 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborrpc/rpc.go @@ -1,6 +1,7 @@ package cborrpc import ( + "bytes" "encoding/hex" "io" @@ -43,3 +44,11 @@ func ReadCborRPC(r io.Reader, out interface{}) error { } return cbor.DecodeReader(r, out) } + +func Dump(obj interface{}) ([]byte, error) { + var out bytes.Buffer + if err := WriteCborRPC(&out, obj); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/storage/miner.go b/storage/miner.go index 35d726d0c..98326e609 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -129,12 +129,12 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal log.Error("seal we just created failed verification") } - params := &actors.CommitSectorParams{ - SectorID: sinfo.SectorID, - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - CommRStar: sinfo.CommRStar[:], - Proof: sinfo.Proof, + params := &actors.OnChainSealVerifyInfo{ + SectorNumber: sinfo.SectorID, + CommD: sinfo.CommD[:], + CommR: sinfo.CommR[:], + CommRStar: sinfo.CommRStar[:], + Proof: sinfo.Proof, } enc, aerr := actors.SerializeParams(params) if aerr != nil { From 213ac77d08f14cab351eb94d780e3d08a8eaaf69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 21:29:05 +0200 Subject: [PATCH 017/230] on chain deals: Expose more chain state in pond --- api/api.go | 2 + api/struct.go | 10 ++++ chain/actors/cbor_gen.go | 58 ++++++++++++++++++ chain/deals/client.go | 4 +- chain/deals/provider.go | 5 +- chain/gen/utils.go | 16 ++--- gen/main.go | 1 + lib/cborrpc/rpc.go | 14 +++++ lotuspond/front/package-lock.json | 14 +++++ lotuspond/front/package.json | 1 + lotuspond/front/src/Address.js | 26 ++++++--- lotuspond/front/src/Block.js | 9 ++- lotuspond/front/src/State.js | 97 +++++++++++++++++++++++++++++-- lotuspond/front/src/chain/code.js | 5 ++ node/impl/full/state.go | 63 ++++++++++++++++++++ 15 files changed, 298 insertions(+), 27 deletions(-) create mode 100644 lotuspond/front/src/chain/code.js diff --git a/api/api.go b/api/api.go index ea444a2c9..1193f1d37 100644 --- a/api/api.go +++ b/api/api.go @@ -135,6 +135,8 @@ type FullNode interface { StateListMiners(context.Context, *types.TipSet) ([]address.Address, error) StateListActors(context.Context, *types.TipSet) ([]address.Address, error) StateMarketBalance(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) + StateMarketParticipants(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) + StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) diff --git a/api/struct.go b/api/struct.go index 1f09733f9..c76af4ed0 100644 --- a/api/struct.go +++ b/api/struct.go @@ -101,6 +101,8 @@ type FullNodeStruct struct { StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` + StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` + StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` @@ -394,6 +396,14 @@ func (c *FullNodeStruct) StateMarketBalance(ctx context.Context, addr address.Ad return c.Internal.StateMarketBalance(ctx, addr, ts) } +func (c *FullNodeStruct) StateMarketParticipants(ctx context.Context, ts *types.TipSet) (map[string]actors.StorageParticipantBalance, error) { + return c.Internal.StateMarketParticipants(ctx, ts) +} + +func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[string]actors.OnChainDeal, error) { + return c.Internal.StateMarketDeals(ctx, ts) +} + func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index f26978564..b1503bfa8 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3476,3 +3476,61 @@ func (t *ProcessStorageDealsPaymentParams) UnmarshalCBOR(r io.Reader) error { return nil } + +func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Deal (actors.StorageDeal) + if err := t.Deal.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ActivationEpoch (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ActivationEpoch)); err != nil { + return err + } + return nil +} + +func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Deal (actors.StorageDeal) + + { + + if err := t.Deal.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ActivationEpoch (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ActivationEpoch = extra + return nil +} diff --git a/chain/deals/client.go b/chain/deals/client.go index 94cb1b8ba..caa47c3ce 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,12 +2,10 @@ package deals import ( "context" - "math" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" @@ -185,7 +183,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, err } - proposalNd, err := cbor.WrapObject(proposal, math.MaxUint64, -1) + proposalNd, err := cborrpc.AsIpld(proposal) if err != nil { return cid.Undef, err } diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 22d94c36b..caf73ca84 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -2,13 +2,11 @@ package deals import ( "context" - "math" "sync" cid "github.com/ipfs/go-cid" datastore "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -17,6 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -192,7 +191,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) { - proposalNd, err := cbor.WrapObject(proposal, math.MaxUint64, -1) + proposalNd, err := cborrpc.AsIpld(proposal) if err != nil { return MinerDeal{}, err } diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 666b9569c..a01b0f20a 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -5,6 +5,14 @@ import ( "fmt" amt "github.com/filecoin-project/go-amt-ipld" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + hamt "github.com/ipfs/go-hamt-ipld" + bstore "github.com/ipfs/go-ipfs-blockstore" + peer "github.com/libp2p/go-libp2p-peer" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/build" actors "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" @@ -12,14 +20,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" - "golang.org/x/xerrors" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - hamt "github.com/ipfs/go-hamt-ipld" - bstore "github.com/ipfs/go-ipfs-blockstore" - peer "github.com/libp2p/go-libp2p-peer" - cbg "github.com/whyrusleeping/cbor-gen" ) type GenesisBootstrap struct { diff --git a/gen/main.go b/gen/main.go index 64eae89ab..127336a0b 100644 --- a/gen/main.go +++ b/gen/main.go @@ -86,6 +86,7 @@ func main() { actors.PublishStorageDealResponse{}, actors.ActivateStorageDealsParams{}, actors.ProcessStorageDealsPaymentParams{}, + actors.OnChainDeal{}, ) if err != nil { fmt.Println(err) diff --git a/lib/cborrpc/rpc.go b/lib/cborrpc/rpc.go index 98a5e71ec..901108457 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborrpc/rpc.go @@ -4,8 +4,10 @@ import ( "bytes" "encoding/hex" "io" + "math" cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" cbg "github.com/whyrusleeping/cbor-gen" ) @@ -52,3 +54,15 @@ func Dump(obj interface{}) ([]byte, error) { } return out.Bytes(), nil } + +// TODO: this is a bit ugly, and this package is not exactly the best place +func AsIpld(obj interface{}) (ipld.Node, error) { + if m, ok := obj.(cbg.CBORMarshaler); ok { + b, err := Dump(m) + if err != nil { + return nil, err + } + return cbor.Decode(b, math.MaxUint64, -1) + } + return cbor.WrapObject(obj, math.MaxUint64, -1) +} diff --git a/lotuspond/front/package-lock.json b/lotuspond/front/package-lock.json index 907aea84d..8df204f2e 100644 --- a/lotuspond/front/package-lock.json +++ b/lotuspond/front/package-lock.json @@ -3457,6 +3457,11 @@ } } }, + "classnames": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.2.6.tgz", + "integrity": "sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==" + }, "clean-css": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.1.tgz", @@ -10663,6 +10668,15 @@ "workbox-webpack-plugin": "4.2.0" } }, + "react-tooltip": { + "version": "3.11.1", + "resolved": "https://registry.npmjs.org/react-tooltip/-/react-tooltip-3.11.1.tgz", + "integrity": "sha512-YCMVlEC2KuHIzOQhPplTK5jmBBwoL+PYJJdJKXj7M/h7oevupd/QSVq6z5U7/ehIGXyHsAqvwpdxexDfyQ0o3A==", + "requires": { + "classnames": "^2.2.5", + "prop-types": "^15.6.0" + } + }, "read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", diff --git a/lotuspond/front/package.json b/lotuspond/front/package.json index a5f397612..c9e6ac769 100644 --- a/lotuspond/front/package.json +++ b/lotuspond/front/package.json @@ -13,6 +13,7 @@ "react-dom": "^16.8.6", "react-router-dom": "^5.0.1", "react-scripts": "3.0.1", + "react-tooltip": "^3.11.1", "rpc-websockets": "^4.5.1", "styled-components": "^3.3.3", "xterm": "^3.14.5", diff --git a/lotuspond/front/src/Address.js b/lotuspond/front/src/Address.js index 6069bd29a..26e4e8d00 100644 --- a/lotuspond/front/src/Address.js +++ b/lotuspond/front/src/Address.js @@ -1,8 +1,9 @@ import React from 'react' import CID from 'cids' -import * as multihash from "multihashes"; -import State from "./State"; -import methods from "./chain/methods"; +import ReactTooltip from 'react-tooltip' +import * as multihash from "multihashes" +import State from "./State" +import methods from "./chain/methods" function truncAddr(addr, len) { if (addr.length > len) { @@ -11,6 +12,17 @@ function truncAddr(addr, len) { return addr } +function filStr(raw) { + if(typeof raw !== 'string') { + return raw + } + if(raw.length < 18) { + raw = '0'.repeat(18 - raw.length) + } + let out = (raw.substring(0, raw.length - 18) + '.' + raw.substring(raw.length - 18, raw.length)).replace(/\.0+|0+$/g, ''); + return out ? out : '0' +} + let sheet = document.createElement('style') document.body.appendChild(sheet); @@ -62,7 +74,7 @@ class Address extends React.Component { } openState() { - this.props.mountWindow((onClose) => ) + this.props.mountWindow((onClose) => ) } async actorInfo(actor) { @@ -117,12 +129,12 @@ class Address extends React.Component { nonce =  Nc:{this.state.nonce}{nonce} } - let balance = : {this.state.balance}  + let balance = : {filStr(this.state.balance)}  if(this.props.nobalance) { balance = } if(this.props.short) { - actInfo = + actInfo = {actInfo}: {filStr(this.state.balance)} balance = } @@ -136,7 +148,7 @@ class Address extends React.Component { minerInfo =  Power: {this.state.minerInfo.MinerPower} ({this.state.minerInfo.MinerPower/this.state.minerInfo.TotalPower*100}%) } - return {addr}{balance}{actInfo}{nonce}{add20k}{transfer}{minerInfo} + return {addr}{balance}{actInfo}{nonce}{add20k}{transfer}{minerInfo} } } diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index 422520034..c7f7ae682 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -64,7 +64,14 @@ class Block extends React.Component {
Miner: {
}
Messages: {head.Messages['/']} {/*TODO: link to message explorer */}
Parent Receipts: {head.ParentMessageReceipts['/']}
-
Parent State Root: {head.ParentStateRoot['/']}
+
+ Parent State Root: {head.ParentStateRoot['/']} +  
+  
+  
+  
+  
+
----
{messages}
diff --git a/lotuspond/front/src/State.js b/lotuspond/front/src/State.js index 70ddbc549..71a2ad909 100644 --- a/lotuspond/front/src/State.js +++ b/lotuspond/front/src/State.js @@ -1,7 +1,17 @@ import React from 'react' import Window from "./Window"; +import CID from "cids"; +import * as multihash from "multihashes"; +import code from "./chain/code"; +import Address from "./Address"; class State extends React.Component { + byCode = { + [code.init]: InitState, + [code.power]: PowerState, + [code.market]: MarketState, + } + constructor(props) { super(props) @@ -9,22 +19,99 @@ class State extends React.Component { } async componentDidMount() { - const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props + const tipset = this.props.tipset || await this.props.client.call("Filecoin.ChainHead", []) const actstate = await this.props.client.call('Filecoin.StateReadState', [this.props.actor, tipset]) - this.setState(actstate) + + const c = new CID(this.props.actor.Code['/']) + const mh = multihash.decode(c.multihash) + let code = mh.digest.toString() + + this.setState({...actstate, code: code}) } render() { + let state + if(this.byCode[this.state.code]) { + const Stelem = this.byCode[this.state.code] + state = + } else { + state =
{Object.keys(this.state.State).map(k =>
{k}: {JSON.stringify(this.state.State[k])}
)}
+ } + const content =
Balance: {this.state.Balance}
---
-
{Object.keys(this.state.State).map(k =>
{k}: {JSON.stringify(this.state.State[k])}
)}
+ {state}
- - return + return {content} } } +class InitState extends React.Component { + constructor(props) { + super(props) + + this.state = {actors: []} + } + + async componentDidMount() { + const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props + const actors = await this.props.client.call("Filecoin.StateListActors", [tipset]) + this.setState({actors: actors}) + } + + render() { + return this.state.actors.sort((a, b) => (Number(a.substr(1)) > Number(b.substr(1)))) + .map(addr =>
) + } +} + +class PowerState extends React.Component { + constructor(props) { + super(props) + + this.state = {actors: []} + } + + async componentDidMount() { + const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props + const actors = await this.props.client.call("Filecoin.StateListMiners", [tipset]) + this.setState({actors: actors}) + } + + render() { + return this.state.actors.sort((a, b) => (Number(a.substr(1)) > Number(b.substr(1)))) + .map(addr =>
) + } +} + +class MarketState extends React.Component { + constructor(props) { + super(props) + this.state = {participants: {}, deals: []} + } + + async componentDidMount() { + const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props + const participants = await this.props.client.call("Filecoin.StateMarketParticipants", [tipset]) + const deals = await this.props.client.call("Filecoin.StateMarketDeals", [tipset]) + this.setState({participants, deals}) + } + + render() { + return
+
+
Participants:
+ {Object.keys(this.state.participants).map(p => {p})} +
+
+
Deals:
+ {this.state.deals.map(d => {d})} +
+
+ } +} + export default State \ No newline at end of file diff --git a/lotuspond/front/src/chain/code.js b/lotuspond/front/src/chain/code.js new file mode 100644 index 000000000..8dd41e0d3 --- /dev/null +++ b/lotuspond/front/src/chain/code.js @@ -0,0 +1,5 @@ +export default { + init: 'filecoin/1.0/InitActor', + power: 'filecoin/1.0/StoragePowerActor', + market: 'filecoin/1.0/StorageMarketActor' +} diff --git a/node/impl/full/state.go b/node/impl/full/state.go index c6a8e2e15..44f0e5220 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1,11 +1,15 @@ package full import ( + "bytes" "context" + "github.com/filecoin-project/go-amt-ipld" + "strconv" cid "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" "go.uber.org/fx" "golang.org/x/xerrors" @@ -240,3 +244,62 @@ func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, return b[0], nil } + +func (a *StateAPI) StateMarketParticipants(ctx context.Context, ts *types.TipSet) (map[string]actors.StorageParticipantBalance, error) { + out := map[string]actors.StorageParticipantBalance{} + + var state actors.StorageMarketState + if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + return nil, err + } + cst := hamt.CSTFromBstore(a.StateManager.ChainStore().Blockstore()) + nd, err := hamt.LoadNode(ctx, cst, state.Balances) + if err != nil { + return nil, err + } + + err = nd.ForEach(ctx, func(k string, val interface{}) error { + cv := val.(*cbg.Deferred) + a, err := address.NewFromBytes([]byte(k)) + if err != nil { + return err + } + var b actors.StorageParticipantBalance + if err := b.UnmarshalCBOR(bytes.NewReader(cv.Raw)); err != nil { + return err + } + out[a.String()] = b + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[string]actors.OnChainDeal, error) { + out := map[string]actors.OnChainDeal{} + + var state actors.StorageMarketState + if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + return nil, err + } + + blks := amt.WrapBlockstore(a.StateManager.ChainStore().Blockstore()) + da, err := amt.LoadAMT(blks, state.Deals) + if err != nil { + return nil, err + } + + if err := da.ForEach(func(i uint64, v *cbg.Deferred) error { + var d actors.OnChainDeal + if err := d.UnmarshalCBOR(bytes.NewReader(v.Raw)); err != nil { + return err + } + out[strconv.FormatInt(int64(i), 10)] = d + return nil + }); err != nil { + return nil, err + } + return out, nil +} From 8e7e5d3085dfd63020d157c6c88d1af535336faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 Oct 2019 21:41:40 +0200 Subject: [PATCH 018/230] on chain deals: Fix deal state serialization --- api/api.go | 1 - chain/deals/state_store.go | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/api.go b/api/api.go index 1193f1d37..e0e7c142e 100644 --- a/api/api.go +++ b/api/api.go @@ -11,7 +11,6 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go index 00c5f0424..32b73835b 100644 --- a/chain/deals/state_store.go +++ b/chain/deals/state_store.go @@ -1,10 +1,11 @@ package deals import ( + "bytes" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" - cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" ) @@ -22,7 +23,7 @@ func (st *StateStore) Begin(i cid.Cid, state interface{}) error { return xerrors.Errorf("Already tracking state for %s", i) } - b, err := cbor.DumpObject(state) + b, err := cborrpc.Dump(state) if err != nil { return err } @@ -76,7 +77,7 @@ func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { return func(in []byte) ([]byte, error) { var deal MinerDeal - err := cbor.DecodeInto(in, &deal) + err := cborrpc.ReadCborRPC(bytes.NewReader(in), &deal) if err != nil { return nil, err } @@ -85,7 +86,7 @@ func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { return nil, err } - return cbor.DumpObject(deal) + return cborrpc.Dump(deal) } } @@ -100,7 +101,7 @@ func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) er func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { return func(in []byte) ([]byte, error) { var deal ClientDeal - err := cbor.DecodeInto(in, &deal) + err := cborrpc.ReadCborRPC(bytes.NewReader(in), &deal) if err != nil { return nil, err } @@ -109,7 +110,7 @@ func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { return nil, err } - return cbor.DumpObject(deal) + return cborrpc.Dump(deal) } } @@ -129,7 +130,7 @@ func (st *ClientStateStore) ListClient() ([]ClientDeal, error) { } var deal ClientDeal - err := cbor.DecodeInto(res.Value, &deal) + err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), &deal) if err != nil { return nil, err } From bd77bba6766ca7097b9375ce7a93f3c122e76e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Oct 2019 12:42:52 +0200 Subject: [PATCH 019/230] on chain deals: Improve some errors --- chain/deals/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chain/deals/client.go b/chain/deals/client.go index caa47c3ce..ac3d5d54c 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -180,23 +180,23 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro } if err := api.SignWith(ctx, c.w.Sign, p.Client, &proposal); err != nil { - return cid.Undef, err + return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } proposalNd, err := cborrpc.AsIpld(proposal) if err != nil { - return cid.Undef, err + return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) if err != nil { s.Reset() - return cid.Undef, err + return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } if err := cborrpc.WriteCborRPC(s, proposal); err != nil { s.Reset() - return cid.Undef, err + return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) } deal := ClientDeal{ From 61e14d0f4c1679ffa424e3cce43a23df0147c06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Oct 2019 14:59:57 +0200 Subject: [PATCH 020/230] on chain deals: Fix some serialization bugs --- chain/actors/actor_storagemarket.go | 2 +- chain/deals/cbor_gen.go | 86 ++++++++++++++++++++++------- chain/deals/client.go | 14 ++--- chain/deals/client_states.go | 4 +- chain/deals/provider.go | 4 +- chain/deals/provider_states.go | 19 ++++--- chain/deals/provider_utils.go | 11 ++-- chain/deals/state_store.go | 12 ++-- chain/deals/types.go | 6 +- node/impl/full/state.go | 6 +- 10 files changed, 106 insertions(+), 58 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 0007fb0b3..651b02b77 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -114,7 +114,7 @@ func (sdp *StorageDealProposal) Verify() error { unsigned := *sdp unsigned.ProposerSignature = nil var buf bytes.Buffer - if err := sdp.MarshalCBOR(&buf); err != nil { + if err := unsigned.MarshalCBOR(&buf); err != nil { return err } diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 9952630e3..41342d56f 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -4,11 +4,11 @@ import ( "fmt" "io" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" "github.com/libp2p/go-libp2p-core/peer" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/chain/types" ) /* This file was generated by github.com/whyrusleeping/cbor-gen */ @@ -165,7 +165,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return err } - // t.t.State (api.DealState) + // t.t.State (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { return err } @@ -191,14 +191,26 @@ func (t *Response) MarshalCBOR(w io.Writer) error { // t.t.PublishMessage (cid.Cid) - if err := cbg.WriteCid(w, t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + if t.PublishMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } } // t.t.CommitMessage (cid.Cid) - if err := cbg.WriteCid(w, t.CommitMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + if t.CommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + } } return nil @@ -219,7 +231,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.State (api.DealState) + // t.t.State (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -255,33 +267,69 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { { - if err := t.StorageDeal.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.StorageDeal = new(actors.StorageDeal) + if err := t.StorageDeal.UnmarshalCBOR(br); err != nil { + return err + } + } } // t.t.PublishMessage (cid.Cid) { - c, err := cbg.ReadCid(br) + pb, err := br.PeekByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { - t.PublishMessage = c + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + } + + t.PublishMessage = &c + } } // t.t.CommitMessage (cid.Cid) { - c, err := cbg.ReadCid(br) + pb, err := br.PeekByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { - t.CommitMessage = c + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = &c + } } return nil @@ -528,7 +576,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.State (api.DealState) + // t.t.State (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { return err } @@ -579,7 +627,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.State (api.DealState) + // t.t.State (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -630,7 +678,7 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.t.State (api.DealState) + // t.t.State (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { return err } @@ -694,7 +742,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.ProposalCid = c } - // t.t.State (api.DealState) + // t.t.State (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { diff --git a/chain/deals/client.go b/chain/deals/client.go index ac3d5d54c..05ac4d60f 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -46,7 +46,7 @@ type Client struct { deals ClientStateStore conns map[cid.Cid]inet.Stream - incoming chan ClientDeal + incoming chan *ClientDeal updated chan clientDealUpdate stop chan struct{} @@ -71,7 +71,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * deals: ClientStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, conns: map[cid.Cid]inet.Stream{}, - incoming: make(chan ClientDeal, 16), + incoming: make(chan *ClientDeal, 16), updated: make(chan clientDealUpdate, 16), stop: make(chan struct{}), @@ -98,7 +98,7 @@ func (c *Client) Run(ctx context.Context) { }() } -func (c *Client) onIncoming(deal ClientDeal) { +func (c *Client) onIncoming(deal *ClientDeal) { log.Info("incoming deal") if _, ok := c.conns[deal.ProposalCid]; ok { @@ -167,7 +167,7 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - proposal := actors.StorageDealProposal{ + proposal := &actors.StorageDealProposal{ PieceRef: p.Data.Bytes(), PieceSize: p.DataSize, PieceSerialization: actors.SerializationUnixFSv0, @@ -179,7 +179,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro StorageCollateral: types.NewInt(p.DataSize), // TODO: real calc } - if err := api.SignWith(ctx, c.w.Sign, p.Client, &proposal); err != nil { + if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil { return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } @@ -199,9 +199,9 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) } - deal := ClientDeal{ + deal := &ClientDeal{ ProposalCid: proposalNd.Cid(), - Proposal: proposal, + Proposal: *proposal, State: api.DealUnknown, Miner: p.MinerID, diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 28164c61f..26f16cc51 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -38,7 +38,7 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) error { } // TODO: spec says it's optional - pubmsg, err := c.chain.GetMessage(resp.PublishMessage) + pubmsg, err := c.chain.GetMessage(*resp.PublishMessage) if err != nil { return xerrors.Errorf("getting deal pubsish message: %w", err) } @@ -56,7 +56,7 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) error { } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, resp.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, *resp.PublishMessage) if err != nil { return xerrors.Errorf("Waiting for deal publish message: %w", err) } diff --git a/chain/deals/provider.go b/chain/deals/provider.go index caf73ca84..69b39cc42 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -141,7 +141,7 @@ func (p *Provider) onIncoming(deal MinerDeal) { p.conns[deal.ProposalCid] = deal.s - if err := p.deals.Begin(deal.ProposalCid, deal); err != nil { + if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil { // This can happen when client re-sends proposal p.failDeal(deal.ProposalCid, err) log.Errorf("deal tracking failed: %s", err) @@ -191,7 +191,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) { - proposalNd, err := cborrpc.AsIpld(proposal) + proposalNd, err := cborrpc.AsIpld(&proposal) if err != nil { return MinerDeal{}, err } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index ed7baea86..02a058c3c 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -80,7 +80,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // check market funds clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) if err != nil { - return nil, err + return nil, xerrors.Errorf("getting client market balance failed: %w", err) } // This doesn't guarantee that the client won't withdraw / lock those funds @@ -89,9 +89,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.New("clientMarketBalance.Available too small") } - providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) + providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Provider, nil) if err != nil { - return nil, err + return nil, xerrors.Errorf("getting provider market balance failed: %w", err) } // TODO: this needs to be atomic @@ -124,11 +124,12 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } log.Info("fetching data for a deal") - err = p.sendSignedResponse(Response{ + mcid := smsg.Cid() + err = p.sendSignedResponse(&Response{ State: api.DealAccepted, Message: "", Proposal: deal.ProposalCid, - PublishMessage: smsg.Cid(), + PublishMessage: &mcid, }) if err != nil { return nil, err @@ -140,7 +141,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // STAGED func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(Response{ + err := p.sendSignedResponse(&Response{ State: api.DealStaged, Proposal: deal.ProposalCid, }) @@ -205,7 +206,7 @@ func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilde } func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(Response{ + err := p.sendSignedResponse(&Response{ State: api.DealSealing, Proposal: deal.ProposalCid, }) @@ -231,11 +232,11 @@ func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDea log.Warnf("Waiting for sector commitment message: %s", err) } - err = p.sendSignedResponse(Response{ + err = p.sendSignedResponse(&Response{ State: api.DealComplete, Proposal: deal.ProposalCid, - CommitMessage: mcid, + CommitMessage: &mcid, }) if err != nil { log.Warnf("Sending deal response failed: %s", err) diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 3a9f96233..5931cefc9 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" ) @@ -29,7 +28,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) - err := p.sendSignedResponse(Response{ + err := p.sendSignedResponse(&Response{ State: api.DealFailed, Message: cerr.Error(), Proposal: id, @@ -67,13 +66,13 @@ func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealPropo return } -func (p *Provider) sendSignedResponse(resp Response) error { +func (p *Provider) sendSignedResponse(resp *Response) error { s, ok := p.conns[resp.Proposal] if !ok { return xerrors.New("couldn't send response: not connected") } - msg, err := cbor.DumpObject(&resp) + msg, err := cborrpc.Dump(resp) if err != nil { return xerrors.Errorf("serializing response: %w", err) } @@ -88,8 +87,8 @@ func (p *Provider) sendSignedResponse(resp Response) error { return xerrors.Errorf("failed to sign response message: %w", err) } - signedResponse := SignedResponse{ - Response: resp, + signedResponse := &SignedResponse{ + Response: *resp, Signature: sig, } diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go index 32b73835b..ae959146f 100644 --- a/chain/deals/state_store.go +++ b/chain/deals/state_store.go @@ -76,13 +76,13 @@ func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { return func(in []byte) ([]byte, error) { - var deal MinerDeal - err := cborrpc.ReadCborRPC(bytes.NewReader(in), &deal) + deal := new(MinerDeal) + err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) if err != nil { return nil, err } - if err := m(&deal); err != nil { + if err := m(deal); err != nil { return nil, err } @@ -100,13 +100,13 @@ func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) er func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { return func(in []byte) ([]byte, error) { - var deal ClientDeal - err := cborrpc.ReadCborRPC(bytes.NewReader(in), &deal) + deal := new(ClientDeal) + err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) if err != nil { return nil, err } - if err := m(&deal); err != nil { + if err := m(deal); err != nil { return nil, err } diff --git a/chain/deals/types.go b/chain/deals/types.go index d4b445328..27bf31a8b 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -23,11 +23,11 @@ type Response struct { Proposal cid.Cid // DealAccepted - StorageDeal actors.StorageDeal - PublishMessage cid.Cid + StorageDeal *actors.StorageDeal + PublishMessage *cid.Cid // DealComplete - CommitMessage cid.Cid + CommitMessage *cid.Cid } // TODO: Do we actually need this to be signed? diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 44f0e5220..115d73d86 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -233,7 +233,7 @@ func (a *StateAPI) StateListActors(ctx context.Context, ts *types.TipSet) ([]add func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (actors.StorageParticipantBalance, error) { var state actors.StorageMarketState - if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + if _, err := a.StateManager.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { return actors.StorageParticipantBalance{}, err } cst := hamt.CSTFromBstore(a.StateManager.ChainStore().Blockstore()) @@ -249,7 +249,7 @@ func (a *StateAPI) StateMarketParticipants(ctx context.Context, ts *types.TipSet out := map[string]actors.StorageParticipantBalance{} var state actors.StorageMarketState - if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + if _, err := a.StateManager.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { return nil, err } cst := hamt.CSTFromBstore(a.StateManager.ChainStore().Blockstore()) @@ -281,7 +281,7 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[ out := map[string]actors.OnChainDeal{} var state actors.StorageMarketState - if _, err := a.StateManager.LoadActorState(ctx, actors.StoragePowerAddress, &state, ts); err != nil { + if _, err := a.StateManager.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { return nil, err } From fabd07416540f963f747951f0effa3a8554ddce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Oct 2019 19:39:14 +0200 Subject: [PATCH 021/230] on chain deals: Deals make it to the chain --- chain/actors/actor_storagemarket.go | 68 +++++++++++++++++++--------- chain/actors/cbor_gen.go | 14 +++++- chain/deals/client.go | 36 ++++++++++++++- chain/deals/client_states.go | 16 +++++-- chain/deals/provider.go | 2 +- chain/deals/provider_asks.go | 2 +- chain/deals/provider_states.go | 41 ++++++++++++++--- chain/gen/gen.go | 4 +- chain/gen/mining.go | 2 +- chain/stmgr/stmgr.go | 14 ++++++ chain/stmgr/utils.go | 21 ++++++++- chain/sync.go | 4 +- lib/jsonrpc/handler.go | 2 +- lotuspond/front/src/Address.js | 3 ++ lotuspond/front/src/State.js | 12 ++++- lotuspond/front/src/chain/methods.js | 26 ++++++++--- node/impl/client/client.go | 2 + node/impl/full/state.go | 32 +------------ 18 files changed, 219 insertions(+), 82 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 651b02b77..9db18e367 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -121,9 +121,31 @@ func (sdp *StorageDealProposal) Verify() error { return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) } +func (d *StorageDeal) Sign(ctx context.Context, sign SignFunc) error { + var buf bytes.Buffer + if err := d.Proposal.MarshalCBOR(&buf); err != nil { + return err + } + sig, err := sign(ctx, buf.Bytes()) + if err != nil { + return err + } + d.CounterSignature = sig + return nil +} + +func (d *StorageDeal) Verify(proposerWorker address.Address) error { + var buf bytes.Buffer + if err := d.Proposal.MarshalCBOR(&buf); err != nil { + return err + } + + return d.CounterSignature.Verify(proposerWorker, buf.Bytes()) +} + type StorageDeal struct { Proposal StorageDealProposal - CounterSignature types.Signature + CounterSignature *types.Signature } type OnChainDeal struct { @@ -214,7 +236,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { for addr, b := range set { - if err := nd.Set(vmctx.Context(), string(addr.Bytes()), b); err != nil { + if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &b); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") } } @@ -283,7 +305,6 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } @@ -298,7 +319,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return nil, err } - err := deals.Set(self.NextDealID, OnChainDeal{Deal: deal}) + err := deals.Set(self.NextDealID, &OnChainDeal{Deal: deal}) if err != nil { return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT") } @@ -338,37 +359,33 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage return aerrors.New(1, "deal proposal already expired") } - var proposalBuf bytes.Buffer - err := deal.Proposal.MarshalCBOR(&proposalBuf) - if err != nil { - return aerrors.HandleExternalError(err, "serializing deal proposal failed") + if err := deal.Proposal.Verify(); err != nil { + return aerrors.Absorb(err, 2, "verifying proposer signature") } - err = deal.Proposal.ProposerSignature.Verify(deal.Proposal.Client, proposalBuf.Bytes()) + workerBytes, aerr := vmctx.Send(deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + if aerr != nil { + return aerr + } + providerWorker, err := address.NewFromBytes(workerBytes) if err != nil { - return aerrors.HandleExternalError(err, "verifying proposer signature") + return aerrors.HandleExternalError(err, "parsing provider worker address bytes") } - var dealBuf bytes.Buffer - err = deal.MarshalCBOR(&dealBuf) + err = deal.Verify(providerWorker) if err != nil { - return aerrors.HandleExternalError(err, "serializing deal failed") - } - - err = deal.CounterSignature.Verify(deal.Proposal.Provider, dealBuf.Bytes()) - if err != nil { - return aerrors.HandleExternalError(err, "verifying provider signature") + return aerrors.Absorb(err, 2, "verifying provider signature") } // TODO: maybe this is actually fine - if vmctx.Message().From != deal.Proposal.Provider && vmctx.Message().From != deal.Proposal.Client { + if vmctx.Message().From != providerWorker && vmctx.Message().From != deal.Proposal.Client { return aerrors.New(4, "message not sent by deal participant") } // TODO: REVIEW: Do we want to check if provider exists in the power actor? // TODO: do some caching (changes gas so needs to be in spec too) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, deal.Proposal.Client, deal.Proposal.Provider) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, deal.Proposal.Client, providerWorker) if aerr != nil { return aerrors.Wrap(aerr, "getting client, and provider balances") } @@ -426,7 +443,16 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types return nil, aerrors.HandleExternalError(err, "getting del info failed") } - if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { + workerBytes, err := vmctx.Send(dealInfo.Deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + if err != nil { + return nil, err + } + providerWorker, eerr := address.NewFromBytes(workerBytes) + if eerr != nil { + return nil, aerrors.HandleExternalError(eerr, "parsing provider worker address bytes") + } + + if vmctx.Message().From != providerWorker { return nil, aerrors.New(1, "ActivateStorageDeals can only be called by deal provider") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index b1503bfa8..48e0ddabc 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3197,9 +3197,21 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { { - if err := t.CounterSignature.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.CounterSignature = new(types.Signature) + if err := t.CounterSignature.UnmarshalCBOR(br); err != nil { + return err + } + } } return nil diff --git a/chain/deals/client.go b/chain/deals/client.go index 05ac4d60f..b43593437 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -42,6 +43,7 @@ type Client struct { w *wallet.Wallet dag dtypes.ClientDAG discovery *discovery.Local + mpool full.MpoolAPI deals ClientStateStore conns map[cid.Cid]inet.Stream @@ -59,7 +61,7 @@ type clientDealUpdate struct { err error } -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local) *Client { +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI) *Client { c := &Client{ sm: sm, chain: chain, @@ -67,6 +69,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * w: w, dag: dag, discovery: discovery, + mpool: mpool, deals: ClientStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, conns: map[cid.Cid]inet.Stream{}, @@ -167,6 +170,37 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { + // check market funds + clientMarketBalance, err := c.sm.MarketBalance(ctx, p.Client, nil) + if err != nil { + return cid.Undef, xerrors.Errorf("getting client market balance failed: %w", err) + } + + if clientMarketBalance.Available.LessThan(p.TotalPrice) { + // TODO: move to a smarter market funds manager + + smsg, err := c.mpool.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: p.Client, + Value: p.TotalPrice, + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return cid.Undef, err + } + + _, r, err := c.sm.WaitForMessage(ctx, smsg.Cid()) + if err != nil { + return cid.Undef, err + } + + if r.ExitCode != 0 { + return cid.Undef, xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode) + } + } + proposal := &actors.StorageDealProposal{ PieceRef: p.Data.Bytes(), PieceSize: p.DataSize, diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 26f16cc51..bf32f57d6 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -4,6 +4,7 @@ import ( "context" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/stmgr" "golang.org/x/xerrors" ) @@ -43,22 +44,27 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) error { return xerrors.Errorf("getting deal pubsish message: %w", err) } - if pubmsg.From != deal.Proposal.Provider { - return xerrors.Errorf("Deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) + if err != nil { + return xerrors.Errorf("getting miner worker failed: %w", err) + } + + if pubmsg.From != pw { + return xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) } if pubmsg.To != actors.StorageMarketAddress { - return xerrors.Errorf("Deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + return xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) } if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { - return xerrors.Errorf("Deal publish message called incorrect method (method=%s)", pubmsg.Method) + return xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) } // TODO: timeout _, ret, err := c.sm.WaitForMessage(ctx, *resp.PublishMessage) if err != nil { - return xerrors.Errorf("Waiting for deal publish message: %w", err) + return xerrors.Errorf("waiting for deal publish message: %w", err) } if ret.ExitCode != 0 { return xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 69b39cc42..0c5a2d210 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -160,7 +160,7 @@ func (p *Provider) onIncoming(deal MinerDeal) { func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { log.Infof("Deal %s updated state to %d", update.id, update.newState) if update.err != nil { - log.Errorf("deal %s failed: %s", update.id, update.err) + log.Errorf("deal %s (newSt: %d) failed: %s", update.id, update.newState, update.err) p.failDeal(update.id, update.err) return } diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index ff1f7f7a1..1b1960b97 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -145,7 +145,7 @@ func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState() - w, err := stmgr.GetMinerWorker(context.TODO(), c.sm, tss, ask.Ask.Miner) + w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner) if err != nil { return xerrors.Errorf("failed to get worker for miner in ask", err) } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 02a058c3c..c2de1c9c0 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -2,15 +2,16 @@ package deals import ( "context" - "github.com/ipfs/go-cid" "github.com/filecoin-project/go-sectorbuilder/sealing_state" + "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -40,11 +41,11 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle // ACCEPTED -func (p *Provider) addMarketFunds(ctx context.Context, deal MinerDeal) error { +func (p *Provider) addMarketFunds(ctx context.Context, worker address.Address, deal MinerDeal) error { log.Info("Adding market funds for storage collateral") smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: deal.Proposal.Provider, + From: worker, Value: deal.Proposal.StorageCollateral, GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), @@ -75,6 +76,14 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.PieceSerialization) } + head, err := p.full.ChainHead(ctx) + if err != nil { + return nil, err + } + if head.Height() >= deal.Proposal.ProposalExpiration { + return nil, xerrors.Errorf("deal proposal already expired") + } + // TODO: check StorageCollateral / StoragePrice // check market funds @@ -89,28 +98,48 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.New("clientMarketBalance.Available too small") } - providerMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Provider, nil) + waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil) + if err != nil { + return nil, err + } + + providerMarketBalance, err := p.full.StateMarketBalance(ctx, waddr, nil) if err != nil { return nil, xerrors.Errorf("getting provider market balance failed: %w", err) } // TODO: this needs to be atomic if providerMarketBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - if err := p.addMarketFunds(ctx, deal); err != nil { + if err := p.addMarketFunds(ctx, waddr, deal); err != nil { return nil, err } } log.Info("publishing deal") + storageDeal := actors.StorageDeal{ + Proposal: deal.Proposal, + } + if err := api.SignWith(ctx, p.full.WalletSign, waddr, &storageDeal); err != nil { + return nil, xerrors.Errorf("signing storage deal failed: ", err) + } + + params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: []actors.StorageDeal{storageDeal}, + }) + if err != nil { + return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) + } + // TODO: We may want this to happen after fetching data smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, - From: deal.Proposal.Provider, + From: waddr, Value: types.NewInt(0), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.PublishStorageDeals, + Params: params, }) if err != nil { return nil, err diff --git a/chain/gen/gen.go b/chain/gen/gen.go index ef1460430..89d52bbce 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -206,7 +206,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add st := pts.ParentState() - worker, err := stmgr.GetMinerWorker(ctx, cg.sm, st, m) + worker, err := stmgr.GetMinerWorkerRaw(ctx, cg.sm, st, m) if err != nil { return nil, nil, err } @@ -385,7 +385,7 @@ func (mca mca) StateMinerPower(ctx context.Context, maddr address.Address, ts *t } func (mca mca) StateMinerWorker(ctx context.Context, maddr address.Address, ts *types.TipSet) (address.Address, error) { - return stmgr.GetMinerWorker(ctx, mca.sm, ts.ParentState(), maddr) + return stmgr.GetMinerWorkerRaw(ctx, mca.sm, ts.ParentState(), maddr) } func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*types.Signature, error) { diff --git a/chain/gen/mining.go b/chain/gen/mining.go index 0ede4460e..5977fc66a 100644 --- a/chain/gen/mining.go +++ b/chain/gen/mining.go @@ -27,7 +27,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal height := parents.Height() + uint64(len(tickets)) - worker, err := stmgr.GetMinerWorker(ctx, sm, st, miner) + worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, st, miner) if err != nil { return nil, xerrors.Errorf("failed to get miner worker: %w", err) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index ecfc51648..14586d2e4 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -459,3 +459,17 @@ func (sm *StateManager) ListAllActors(ctx context.Context, ts *types.TipSet) ([] return out, nil } + +func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (actors.StorageParticipantBalance, error) { + var state actors.StorageMarketState + if _, err := sm.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { + return actors.StorageParticipantBalance{}, err + } + cst := hamt.CSTFromBstore(sm.ChainStore().Blockstore()) + b, _, err := actors.GetMarketBalances(ctx, cst, state.Balances, addr) + if err != nil { + return actors.StorageParticipantBalance{}, err + } + + return b[0], nil +} diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 8a7398b9a..af3787766 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -16,7 +16,7 @@ import ( "golang.org/x/xerrors" ) -func GetMinerWorker(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { +func GetMinerWorkerRaw(ctx context.Context, sm *StateManager, st cid.Cid, maddr address.Address) (address.Address, error) { recp, err := sm.CallRaw(ctx, &types.Message{ To: maddr, From: maddr, @@ -118,7 +118,7 @@ func GetMinerPeerID(ctx context.Context, sm *StateManager, ts *types.TipSet, mad Method: actors.MAMethods.GetPeerID, }, ts) if err != nil { - return "", xerrors.Errorf("callRaw failed: %w", err) + return "", xerrors.Errorf("call failed: %w", err) } if recp.ExitCode != 0 { @@ -128,6 +128,23 @@ func GetMinerPeerID(ctx context.Context, sm *StateManager, ts *types.TipSet, mad return peer.IDFromBytes(recp.Return) } +func GetMinerWorker(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (address.Address, error) { + recp, err := sm.Call(ctx, &types.Message{ + To: maddr, + From: maddr, + Method: actors.MAMethods.GetWorkerAddr, + }, ts) + if err != nil { + return address.Undef, xerrors.Errorf("call failed: %w", err) + } + + if recp.ExitCode != 0 { + return address.Undef, xerrors.Errorf("getting miner peer ID failed (exit code %d)", recp.ExitCode) + } + + return address.NewFromBytes(recp.Return) +} + func GetMinerProvingPeriodEnd(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) { var mas actors.StorageMinerActorState _, err := sm.LoadActorState(ctx, maddr, &mas, ts) diff --git a/chain/sync.go b/chain/sync.go index c27532e09..40af03964 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -482,9 +482,9 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("minerIsValid failed: %w", err) } - waddr, err := stmgr.GetMinerWorker(ctx, syncer.sm, stateroot, h.Miner) + waddr, err := stmgr.GetMinerWorkerRaw(ctx, syncer.sm, stateroot, h.Miner) if err != nil { - return xerrors.Errorf("GetMinerWorker failed: %w", err) + return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err) } if err := h.CheckBlockSignature(ctx, waddr); err != nil { diff --git a/lib/jsonrpc/handler.go b/lib/jsonrpc/handler.go index f08d8da20..e771b172e 100644 --- a/lib/jsonrpc/handler.go +++ b/lib/jsonrpc/handler.go @@ -212,7 +212,7 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer if handler.errOut != -1 { err := callResult[handler.errOut].Interface() if err != nil { - log.Warnf("error in RPC call to '%s': %s", req.Method, err) + log.Warnf("error in RPC call to '%s': %+v", req.Method, err) resp.Error = &respError{ Code: 1, Message: err.(error).Error(), diff --git a/lotuspond/front/src/Address.js b/lotuspond/front/src/Address.js index 26e4e8d00..71eae0979 100644 --- a/lotuspond/front/src/Address.js +++ b/lotuspond/front/src/Address.js @@ -6,6 +6,9 @@ import State from "./State" import methods from "./chain/methods" function truncAddr(addr, len) { + if (!addr) { + return "" + } if (addr.length > len) { return {addr.substr(0, len - 3) + '..'} } diff --git a/lotuspond/front/src/State.js b/lotuspond/front/src/State.js index 71a2ad909..5051023c2 100644 --- a/lotuspond/front/src/State.js +++ b/lotuspond/front/src/State.js @@ -104,11 +104,19 @@ class MarketState extends React.Component { return
Participants:
- {Object.keys(this.state.participants).map(p => {p})} + + + {Object.keys(this.state.participants).map(p => + + + + )} +
AddressAvailableLocked
{this.state.participants[p].Available}{this.state.participants[p].Locked}
+
---
Deals:
- {this.state.deals.map(d => {d})} + {Object.keys(this.state.deals).map(d =>
{d}
)}
} diff --git a/lotuspond/front/src/chain/methods.js b/lotuspond/front/src/chain/methods.js index 3cce9098d..426f7a960 100644 --- a/lotuspond/front/src/chain/methods.js +++ b/lotuspond/front/src/chain/methods.js @@ -1,10 +1,10 @@ export default { - "account": [ + "filecoin/1.0/AccountActor": [ "Send", "Constructor", "GetAddress", ], - "smarket": [ + "filecoin/1.0/StoragePowerActor": [ "Send", "Constructor", "CreateStorageMiner", @@ -15,7 +15,21 @@ export default { "IsMiner", "StorageCollateralForSize" ], - "sminer": [ + "filecoin/1.0/StorageMarketActor": [ + "Send", + "Constructor", + "WithdrawBalance", + "AddBalance", + "CheckLockedBalance", + "PublishStorageDeals", + "HandleCronAction", + "SettleExpiredDeals", + "ProcessStorageDealsPayment", + "SlashStorageDealCollateral", + "GetLastExpirationFromDealIDs", + "ActivateStorageDeals", + ], + "filecoin/1.0/StorageMinerActor": [ "Send", "Constructor", "CommitSector", @@ -36,7 +50,7 @@ export default { "PaymentVerifyInclusion", "PaymentVerifySector", ], - "multisig": [ + "filecoin/1.0/MultisigActor": [ "Send", "Constructor", "Propose", @@ -48,13 +62,13 @@ export default { "SwapSigner", "ChangeRequirement", ], - "init": [ + "filecoin/1.0/InitActor": [ "Send", "Constructor", "Exec", "GetIdForAddress" ], - "paych": [ + "filecoin/1.0/PaymentChannelActor": [ "Send", "Constructor", "UpdateChannelState", diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a5c1ecb81..b6103d3d7 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -5,6 +5,7 @@ import ( "errors" "golang.org/x/xerrors" "io" + "math" "os" "github.com/ipfs/go-blockservice" @@ -81,6 +82,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A proposal := deals.ClientDealProposal{ Data: data, TotalPrice: total, + ProposalExpiration: math.MaxUint64, // TODO: set something reasonable Duration: blocksDuration, ProviderAddress: miner, Client: self, diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 115d73d86..6cad3d648 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -58,25 +58,7 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, maddr address.Address, t } func (a *StateAPI) StateMinerWorker(ctx context.Context, m address.Address, ts *types.TipSet) (address.Address, error) { - ret, err := a.StateManager.Call(ctx, &types.Message{ - From: m, - To: m, - Method: actors.MAMethods.GetWorkerAddr, - }, ts) - if err != nil { - return address.Undef, xerrors.Errorf("failed to get miner worker addr: %w", err) - } - - if ret.ExitCode != 0 { - return address.Undef, xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.ExitCode) - } - - w, err := address.NewFromBytes(ret.Return) - if err != nil { - return address.Undef, xerrors.Errorf("GetWorkerAddr returned malformed address: %w", err) - } - - return w, nil + return stmgr.GetMinerWorker(ctx, a.StateManager, ts, m) } func (a *StateAPI) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) { @@ -232,17 +214,7 @@ func (a *StateAPI) StateListActors(ctx context.Context, ts *types.TipSet) ([]add } func (a *StateAPI) StateMarketBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (actors.StorageParticipantBalance, error) { - var state actors.StorageMarketState - if _, err := a.StateManager.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { - return actors.StorageParticipantBalance{}, err - } - cst := hamt.CSTFromBstore(a.StateManager.ChainStore().Blockstore()) - b, _, err := actors.GetMarketBalances(ctx, cst, state.Balances, addr) - if err != nil { - return actors.StorageParticipantBalance{}, err - } - - return b[0], nil + return a.StateManager.MarketBalance(ctx, addr, ts) } func (a *StateAPI) StateMarketParticipants(ctx context.Context, ts *types.TipSet) (map[string]actors.StorageParticipantBalance, error) { From cdd91914b9ad07a439af123e6de7c33206d9da84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 Oct 2019 20:04:07 +0200 Subject: [PATCH 022/230] on chain deals: Actually set DataSize --- chain/deals/cbor_gen.go | 19 ++----------------- chain/deals/client.go | 9 +++++---- chain/deals/client_utils.go | 25 +++++++++++++++++++++++++ chain/deals/provider_states.go | 10 ++++++++++ lotuspond/front/src/ChainExplorer.js | 8 ++++++-- node/impl/client/client.go | 12 ++++++------ 6 files changed, 54 insertions(+), 29 deletions(-) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 41342d56f..23a79eabb 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -409,7 +409,7 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{136}); err != nil { + if _, err := w.Write([]byte{135}); err != nil { return err } @@ -419,11 +419,6 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Data: %w", err) } - // t.t.DataSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DataSize)); err != nil { - return err - } - // t.t.TotalPrice (types.BigInt) if err := t.TotalPrice.MarshalCBOR(w); err != nil { return err @@ -470,7 +465,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 8 { + if extra != 7 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -486,16 +481,6 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { t.Data = c } - // t.t.DataSize (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DataSize = extra // t.t.TotalPrice (types.BigInt) { diff --git a/chain/deals/client.go b/chain/deals/client.go index b43593437..b6e0b3df5 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -157,8 +157,7 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { } type ClientDealProposal struct { - Data cid.Cid - DataSize uint64 + Data cid.Cid TotalPrice types.BigInt ProposalExpiration uint64 @@ -201,16 +200,18 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro } } + dataSize, err := c.dataSize(ctx, p.Data) + proposal := &actors.StorageDealProposal{ PieceRef: p.Data.Bytes(), - PieceSize: p.DataSize, + PieceSize: uint64(dataSize), PieceSerialization: actors.SerializationUnixFSv0, Client: p.Client, Provider: p.ProviderAddress, ProposalExpiration: p.ProposalExpiration, Duration: p.Duration, StoragePrice: p.TotalPrice, - StorageCollateral: types.NewInt(p.DataSize), // TODO: real calc + StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc } if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil { diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index eb05cb8ca..9c26f4268 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -1,9 +1,12 @@ package deals import ( + "context" "runtime" "github.com/ipfs/go-cid" + files "github.com/ipfs/go-ipfs-files" + unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/lib/cborrpc" @@ -25,6 +28,28 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %s", id, cerr) } +func (c *Client) dataSize(ctx context.Context, data cid.Cid) (int64, error) { + root, err := c.dag.Get(ctx, data) + if err != nil { + log.Errorf("failed to get file root for deal: %s", err) + return 0, err + } + + n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) + if err != nil { + log.Errorf("cannot open unixfs file: %s", err) + return 0, err + } + + uf, ok := n.(files.File) + if !ok { + // TODO: we probably got directory, how should we handle this in unixfs mode? + return 0, xerrors.New("unsupported unixfs type") + } + + return uf.Size() +} + func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { s, ok := c.conns[deal.ProposalCid] if !ok { diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index c2de1c9c0..c604d004b 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -195,6 +195,16 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.Errorf("unsupported unixfs file type") } + // TODO: uf.Size() is user input, not trusted + // This won't be useful / here after we migrate to putting CARs into sectors + size, err := uf.Size() + if err != nil { + return nil, xerrors.Errorf("getting unixfs file size: %w", err) + } + if uint64(size) != deal.Proposal.PieceSize { + return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match unixfs file size") + } + pcid, err := cid.Cast(deal.Proposal.PieceRef) if err != nil { return nil, err diff --git a/lotuspond/front/src/ChainExplorer.js b/lotuspond/front/src/ChainExplorer.js index aec6697bc..1f8f619c0 100644 --- a/lotuspond/front/src/ChainExplorer.js +++ b/lotuspond/front/src/ChainExplorer.js @@ -71,10 +71,10 @@ class ChainExplorer extends React.Component { return } if(!base.Blocks) { - console.log("base for H is nll blk", h, base) + console.log("base for H is nil blk", h, base) return } - let cids = base.Blocks.map(b => b.Parents) + let cids = base.Blocks.map(b => (b.Parents || [])) .reduce((acc, val) => { let out = {...acc} val.forEach(c => out[c['/']] = 8) @@ -85,6 +85,10 @@ class ChainExplorer extends React.Component { const blocks = await Promise.all(cids.map(cid => this.props.client.call('Filecoin.ChainGetBlock', [cid]))) + if (!blocks[0]) { + return + } + cache[h] = { Height: blocks[0].Height, Cids: cids, diff --git a/node/impl/client/client.go b/node/impl/client/client.go index b6103d3d7..926d6f623 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -80,13 +80,13 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A total := types.BigMul(price, types.NewInt(blocksDuration)) proposal := deals.ClientDealProposal{ - Data: data, - TotalPrice: total, + Data: data, + TotalPrice: total, ProposalExpiration: math.MaxUint64, // TODO: set something reasonable - Duration: blocksDuration, - ProviderAddress: miner, - Client: self, - MinerID: pid, + Duration: blocksDuration, + ProviderAddress: miner, + Client: self, + MinerID: pid, } c, err := a.DealClient.Start(ctx, proposal) From 2a49005ed694cfc3f3e977708d1bcc2c51a04fd3 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Oct 2019 14:24:58 +0800 Subject: [PATCH 023/230] cache block message cids to make blocksync cheaper --- chain/store/store.go | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 807188c85..3bbdaa0f7 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -17,6 +17,7 @@ import ( amt "github.com/filecoin-project/go-amt-ipld" "github.com/filecoin-project/lotus/chain/types" + lru "github.com/hashicorp/golang-lru" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" @@ -49,14 +50,18 @@ type ChainStore struct { reorgCh chan<- reorg headChangeNotifs []func(rev, app []*types.TipSet) error + + mmCache *lru.ARCCache } func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore { + c, _ := lru.NewARC(2048) cs := &ChainStore{ bs: bs, ds: ds, bestTips: pubsub.New(64), tipsets: make(map[uint64][]cid.Cid), + mmCache: c, } cs.reorgCh = cs.reorgWorker(context.TODO()) @@ -627,10 +632,21 @@ func (cs *ChainStore) MessagesForTipset(ts *types.TipSet) ([]ChainMsg, error) { return out, nil } -func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { +type mmCids struct { + bls []cid.Cid + secpk []cid.Cid +} + +func (cs *ChainStore) readMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) { + o, ok := cs.mmCache.Get(mmc) + if ok { + mmcids := o.(*mmCids) + return mmcids.bls, mmcids.secpk, nil + } + cst := hamt.CSTFromBstore(cs.bs) var msgmeta types.MsgMeta - if err := cst.Get(context.TODO(), b.Messages, &msgmeta); err != nil { + if err := cst.Get(context.TODO(), mmc, &msgmeta); err != nil { return nil, nil, xerrors.Errorf("failed to load msgmeta: %w", err) } @@ -644,6 +660,20 @@ func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, return nil, nil, errors.Wrap(err, "loading secpk message cids for block") } + cs.mmCache.Add(mmc, &mmCids{ + bls: blscids, + secpk: secpkcids, + }) + + return blscids, secpkcids, nil +} + +func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) { + blscids, secpkcids, err := cs.readMsgMetaCids(b.Messages) + if err != nil { + return nil, nil, err + } + blsmsgs, err := cs.LoadMessagesFromCids(blscids) if err != nil { return nil, nil, errors.Wrap(err, "loading bls messages for block") From acb7d57794512b59f38af61281dffb19bcdf131b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Oct 2019 15:55:00 +0800 Subject: [PATCH 024/230] fix handling of actor not found for balance checks --- chain/stmgr/stmgr.go | 5 ++++- chain/types/fil.go | 2 +- cli/wallet.go | 6 +----- go.sum | 6 ++++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index dd288b743..5fac172f4 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -217,7 +217,10 @@ func (sm *StateManager) GetActor(addr address.Address, ts *types.TipSet) (*types func (sm *StateManager) GetBalance(addr address.Address, ts *types.TipSet) (types.BigInt, error) { act, err := sm.GetActor(addr, ts) if err != nil { - return types.NewInt(0), xerrors.Errorf("get actor: %w", err) + if xerrors.Is(err, types.ErrActorNotFound) { + return types.NewInt(0), nil + } + return types.EmptyInt, xerrors.Errorf("get actor: %w", err) } return act.Balance, nil diff --git a/chain/types/fil.go b/chain/types/fil.go index aed01f2b2..c882c1852 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,7 +12,7 @@ type FIL BigInt func (f FIL) String() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(build.FilecoinPrecision)) - return strings.TrimRight(r.FloatString(18), "0") + return strings.TrimRight(r.FloatString(18), "0.") } func ParseFIL(s string) (FIL, error) { diff --git a/cli/wallet.go b/cli/wallet.go index 4af3eb0f5..cb0cbbdd3 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -3,7 +3,6 @@ package cli import ( "encoding/hex" "encoding/json" - "errors" "fmt" "io/ioutil" "os" @@ -102,10 +101,7 @@ var walletBalance = &cli.Command{ } balance, err := api.WalletBalance(ctx, addr) - - if errors.Is(err, types.ErrActorNotFound) { - log.Warnf("actor not found with address: %s", addr) - } else if err != nil { + if err != nil { return err } diff --git a/go.sum b/go.sum index ec40b61b1..be3b0270d 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -19,6 +20,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -242,6 +244,7 @@ github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsj github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -437,6 +440,7 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -506,7 +510,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= From 34fcbc0bf2047ddd4da8dd5bb653ed8d24d7d1f0 Mon Sep 17 00:00:00 2001 From: waynewyang Date: Thu, 24 Oct 2019 17:13:38 +0800 Subject: [PATCH 025/230] safeguard for multisig actor --- chain/actors/actor_multisig.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/chain/actors/actor_multisig.go b/chain/actors/actor_multisig.go index 1ddb9d6a3..71a6d0e09 100644 --- a/chain/actors/actor_multisig.go +++ b/chain/actors/actor_multisig.go @@ -29,11 +29,11 @@ func (msas MultiSigActorState) isSigner(addr address.Address) bool { return false } -func (msas MultiSigActorState) getTransaction(txid uint64) *MTransaction { - if txid > uint64(len(msas.Transactions)) { - return nil +func (msas MultiSigActorState) getTransaction(txid uint64) (*MTransaction, ActorError) { + if txid >= uint64(len(msas.Transactions)) { + return nil, aerrors.Newf(1, "could not get transaction (numbers of tx %d,want to get txid %d)", len(msas.Transactions), txid) } - return &msas.Transactions[txid] + return &msas.Transactions[txid], nil } type MTransaction struct { @@ -176,7 +176,11 @@ func (msa MultiSigActor) Propose(act *types.Actor, vmctx types.VMContext, } self.Transactions = append(self.Transactions, tx) } - tx := self.getTransaction(txid) + + tx, err := self.getTransaction(txid) + if err != nil { + return nil, err + } if self.Required == 1 { _, err := vmctx.Send(tx.To, tx.Method, tx.Value, tx.Params) @@ -209,7 +213,11 @@ func (msa MultiSigActor) Approve(act *types.Actor, vmctx types.VMContext, return nil, err } - tx := self.getTransaction(params.TxID) + tx, err := self.getTransaction(params.TxID) + if err != nil { + return nil, err + } + if err := tx.Active(); err != nil { return nil, aerrors.Wrap(err, "could not approve") } @@ -240,7 +248,11 @@ func (msa MultiSigActor) Cancel(act *types.Actor, vmctx types.VMContext, return nil, err } - tx := self.getTransaction(params.TxID) + tx, err := self.getTransaction(params.TxID) + if err != nil { + return nil, err + } + if err := tx.Active(); err != nil { return nil, aerrors.Wrap(err, "could not cancel") } @@ -376,6 +388,11 @@ func (msa MultiSigActor) ChangeRequirement(act *types.Actor, vmctx types.VMConte if params.Req < 1 { return nil, aerrors.New(5, "requirement must be at least 1") } + + if params.Req > uint64(len(self.Signers)) { + return nil, aerrors.New(6, "requirement must be at most the numbers of signers") + } + self.Required = params.Req return nil, msa.save(vmctx, head, self) } From 7c78105df529a86061d199e48f40d0cd9a3ecea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 16:22:30 +0200 Subject: [PATCH 026/230] on chain deals: Better deal display in SMA --- lotuspond/front/src/Client.js | 6 ++---- lotuspond/front/src/State.js | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lotuspond/front/src/Client.js b/lotuspond/front/src/Client.js index 56aa82ffe..1c669d193 100644 --- a/lotuspond/front/src/Client.js +++ b/lotuspond/front/src/Client.js @@ -6,13 +6,11 @@ const dealStates = [ "Unknown", "Rejected", "Accepted", - "Started", - "Failed", "Staged", "Sealing", + "Failed", "Complete", "Error", - "Expired" ] @@ -83,7 +81,7 @@ class Client extends React.Component { let deals = this.state.deals.map((deal, i) =>
    -
  • {i}. Proposal: {deal.ProposalCid['/'].substr(0, 18)}...
    : {dealStates[deal.State]} +
  • {i}. Proposal: {deal.ProposalCid['/'].substr(0, 18)}...
    : {dealStates[deal.State]} {dealStates[deal.State] === 'Complete' ?  [Retrieve] : }
    • Data: {deal.PieceRef['/']}, {deal.Size}B; Duration: {deal.Duration}Blocks
      • diff --git a/lotuspond/front/src/State.js b/lotuspond/front/src/State.js index 5051023c2..e1fb42609 100644 --- a/lotuspond/front/src/State.js +++ b/lotuspond/front/src/State.js @@ -116,7 +116,18 @@ class MarketState extends React.Component {
      ---
      Deals:
      - {Object.keys(this.state.deals).map(d =>
      {d}
      )} + + + {Object.keys(this.state.deals).map(d => + + + + + + + + )} +
      idActiveClientProviderSizePriceDuration
      {d}{this.state.deals[d].ActivationEpoch || "No"}
      		{this.state.deals[d].Deal.Proposal.PieceSize}B{this.state.deals[d].Deal.Proposal.StoragePrice}{this.state.deals[d].Deal.Proposal.Duration}
} From 76f1e6e207a19038709dea2204fa16a93d0feb0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 16:24:31 +0200 Subject: [PATCH 027/230] on chain deals: Put dealIDs in CommitSector messages --- api/types.go | 2 - chain/actors/actor_storagemarket.go | 35 +++---- chain/deals/cbor_gen.go | 19 +++- chain/deals/provider.go | 1 + chain/deals/provider_states.go | 14 ++- lib/sectorbuilder/sectorbuilder_test.go | 4 +- node/impl/storminer.go | 3 +- storage/commitment/tracker.go | 27 ++--- storage/miner.go | 16 ++- storage/sector/store.go | 129 +++++++++++++++++++----- storage/sectorblocks/blocks.go | 4 +- 11 files changed, 187 insertions(+), 67 deletions(-) diff --git a/api/types.go b/api/types.go index 475eace60..6a5153328 100644 --- a/api/types.go +++ b/api/types.go @@ -16,13 +16,11 @@ const ( DealSealing // Data in process of being sealed DealFailed - DealComplete // Internal DealError // deal failed with an unexpected error - DealExpired DealNoUpdate = DealUnknown ) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 9db18e367..2bd982bd7 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -408,8 +408,8 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage // TODO: piece checks (e.g. size > sectorSize)? bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - deal.Proposal.Client: clientBalance, - deal.Proposal.Provider: providerBalance, + deal.Proposal.Client: clientBalance, + providerWorker: providerBalance, }) if aerr != nil { return aerr @@ -443,17 +443,8 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types return nil, aerrors.HandleExternalError(err, "getting del info failed") } - workerBytes, err := vmctx.Send(dealInfo.Deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) - if err != nil { - return nil, err - } - providerWorker, eerr := address.NewFromBytes(workerBytes) - if eerr != nil { - return nil, aerrors.HandleExternalError(eerr, "parsing provider worker address bytes") - } - - if vmctx.Message().From != providerWorker { - return nil, aerrors.New(1, "ActivateStorageDeals can only be called by deal provider") + if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { + return nil, aerrors.New(1, "ActivateStorageDeals can only be called by the deal provider") } if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { @@ -467,7 +458,7 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types dealInfo.ActivationEpoch = vmctx.BlockHeight() - if err := deals.Set(deal, dealInfo); err != nil { + if err := deals.Set(deal, &dealInfo); err != nil { return nil, aerrors.HandleExternalError(err, "setting deal info in AMT failed") } } @@ -540,7 +531,17 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // TODO: division is hard, this more than likely has some off-by-one issue toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealInfo.Deal.Proposal.Duration)) - b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, dealInfo.Deal.Proposal.Provider) + // TODO: cache somehow to conserve gas + workerBytes, err := vmctx.Send(dealInfo.Deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + if err != nil { + return nil, err + } + providerWorker, eerr := address.NewFromBytes(workerBytes) + if eerr != nil { + return nil, aerrors.HandleExternalError(eerr, "parsing provider worker address bytes") + } + + b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) clientBal := b[0] providerBal := b[1] @@ -548,8 +549,8 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // TODO: call set once bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - dealInfo.Deal.Proposal.Client: clientBal, - dealInfo.Deal.Proposal.Provider: providerBal, + dealInfo.Deal.Proposal.Client: clientBal, + providerWorker: providerBal, }) if aerr != nil { return nil, aerr diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 23a79eabb..0e09cc492 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -640,7 +640,7 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{135}); err != nil { return err } @@ -674,6 +674,11 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Ref: %w", err) } + // t.t.DealID (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DealID)); err != nil { + return err + } + // t.t.SectorID (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { return err @@ -692,7 +697,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 7 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -749,6 +754,16 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.Ref = c } + // t.t.DealID (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = extra // t.t.SectorID (uint64) maj, extra, err = cbg.CborReadHeader(br) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 0c5a2d210..5a3c0ab65 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -29,6 +29,7 @@ type MinerDeal struct { Ref cid.Cid + DealID uint64 SectorID uint64 // Set when State >= DealStaged s inet.Stream diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index c604d004b..f0261180e 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -1,6 +1,7 @@ package deals import ( + "bytes" "context" "github.com/filecoin-project/go-sectorbuilder/sealing_state" @@ -151,6 +152,13 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) if r.Receipt.ExitCode != 0 { return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode) } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + return nil, err + } + if len(resp.DealIDs) != 1 { + return nil, xerrors.Errorf("got unexpected number of DealIDs from") + } log.Info("fetching data for a deal") mcid := smsg.Cid() @@ -164,7 +172,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - return nil, merkledag.FetchGraph(ctx, deal.Ref, p.dag) + return func(deal *MinerDeal) { + deal.DealID = resp.DealIDs[0] + }, merkledag.FetchGraph(ctx, deal.Ref, p.dag) } // STAGED @@ -210,7 +220,7 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.Proposal.Duration) + sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.DealID) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 19119ce09..29891fde6 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -6,6 +6,8 @@ import ( "math/rand" "testing" + "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sector" @@ -40,7 +42,7 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - store := sector.NewStore(sb) + store := sector.NewStore(sb, datastore.NewMapDatastore()) store.Service() ssinfo := <-store.Incoming() diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 3ced1e3e2..526dd9fbf 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -33,8 +33,9 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { size := sectorbuilder.UserBytesForSectorSize(build.SectorSize) + // TODO: create a deal name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) + sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), 0) if err != nil { return 0, err } diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go index ea7cb1515..aa50c705e 100644 --- a/storage/commitment/tracker.go +++ b/storage/commitment/tracker.go @@ -25,7 +25,7 @@ func init() { var commitmentDsPrefix = datastore.NewKey("/commitments") type Tracker struct { - commitDs datastore.Datastore + commitments datastore.Datastore lk sync.Mutex @@ -34,35 +34,36 @@ type Tracker struct { func NewTracker(ds dtypes.MetadataDS) *Tracker { return &Tracker{ - commitDs: namespace.Wrap(ds, commitmentDsPrefix), - waits: map[datastore.Key]chan struct{}{}, + commitments: namespace.Wrap(ds, commitmentDsPrefix), + waits: map[datastore.Key]chan struct{}{}, } } type commitment struct { - Msg cid.Cid + DealIDs []uint64 + Msg cid.Cid } func commitmentKey(miner address.Address, sectorId uint64) datastore.Key { return commitmentDsPrefix.ChildString(miner.String()).ChildString(fmt.Sprintf("%d", sectorId)) } -func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, mcid cid.Cid) error { +func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, commitMsg cid.Cid) error { key := commitmentKey(miner, sectorId) ct.lk.Lock() defer ct.lk.Unlock() - tracking, err := ct.commitDs.Get(key) + tracking, err := ct.commitments.Get(key) switch err { case datastore.ErrNotFound: - comm := &commitment{Msg: mcid} + comm := &commitment{Msg: commitMsg} commB, err := cbor.DumpObject(comm) if err != nil { return err } - if err := ct.commitDs.Put(key, commB); err != nil { + if err := ct.commitments.Put(key, commB); err != nil { return err } @@ -78,11 +79,11 @@ func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, return err } - if !comm.Msg.Equals(mcid) { - return xerrors.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, mcid) + if !comm.Msg.Equals(commitMsg) { + return xerrors.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg) } - log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, mcid) + log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg) return nil default: return err @@ -94,7 +95,7 @@ func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sector ct.lk.Lock() - tracking, err := ct.commitDs.Get(key) + tracking, err := ct.commitments.Get(key) if err != datastore.ErrNotFound { ct.lk.Unlock() @@ -120,7 +121,7 @@ func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sector select { case <-wait: - tracking, err := ct.commitDs.Get(key) + tracking, err := ct.commitments.Get(key) if err != nil { return cid.Undef, xerrors.Errorf("failed to get commitment after waiting: %w", err) } diff --git a/storage/miner.go b/storage/miner.go index 98326e609..659a09c59 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -9,6 +9,7 @@ import ( logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" @@ -129,12 +130,19 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal log.Error("seal we just created failed verification") } + deals, err := m.secst.DealsForCommit(sinfo.SectorID) + if err != nil { + return xerrors.Errorf("getting sector deals failed: %w", err) + } + params := &actors.OnChainSealVerifyInfo{ + CommD: sinfo.CommD[:], + CommR: sinfo.CommR[:], + CommRStar: sinfo.CommRStar[:], + Proof: sinfo.Proof, + + DealIDs: deals, SectorNumber: sinfo.SectorID, - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - CommRStar: sinfo.CommRStar[:], - Proof: sinfo.Proof, } enc, aerr := actors.SerializeParams(params) if aerr != nil { diff --git a/storage/sector/store.go b/storage/sector/store.go index 19ca85363..65ccd65ce 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -2,25 +2,45 @@ package sector import ( "context" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" - "golang.org/x/xerrors" + "fmt" "io" "sync" "time" + "github.com/filecoin-project/go-sectorbuilder/sealing_state" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/sectorbuilder" - - logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) +func init() { + cbor.RegisterCborType(dealMapping{}) +} + var log = logging.Logger("sectorstore") +var sectorDealsPrefix = datastore.NewKey("/sectordeals") + +type dealMapping struct { + DealIDs []uint64 + Committed bool +} + // TODO: eventually handle sector storage here instead of in rust-sectorbuilder type Store struct { - lk sync.Mutex + waitingLk sync.Mutex + sb *sectorbuilder.SectorBuilder + dealsLk sync.Mutex + deals datastore.Datastore + waiting map[uint64]chan struct{} incoming []chan sectorbuilder.SectorSealingStatus // TODO: outdated chan @@ -28,9 +48,10 @@ type Store struct { closeCh chan struct{} } -func NewStore(sb *sectorbuilder.SectorBuilder) *Store { +func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS) *Store { return &Store{ sb: sb, + deals: namespace.Wrap(ds, sectorDealsPrefix), waiting: map[uint64]chan struct{}{}, closeCh: make(chan struct{}), } @@ -44,13 +65,13 @@ func (s *Store) poll() { log.Debug("polling for sealed sectors...") // get a list of sectors to poll - s.lk.Lock() + s.waitingLk.Lock() toPoll := make([]uint64, 0, len(s.waiting)) for id := range s.waiting { toPoll = append(toPoll, id) } - s.lk.Unlock() + s.waitingLk.Unlock() var done []sectorbuilder.SectorSealingStatus @@ -68,7 +89,7 @@ func (s *Store) poll() { } // send updates - s.lk.Lock() + s.waitingLk.Lock() for _, sector := range done { watch, ok := s.waiting[sector.SectorID] if ok { @@ -79,7 +100,7 @@ func (s *Store) poll() { c <- sector // TODO: ctx! } } - s.lk.Unlock() + s.waitingLk.Unlock() } func (s *Store) service() { @@ -90,35 +111,97 @@ func (s *Store) service() { case <-poll: s.poll() case <-s.closeCh: - s.lk.Lock() + s.waitingLk.Lock() for _, c := range s.incoming { close(c) } - s.lk.Unlock() + s.waitingLk.Unlock() return } } } -func (s *Store) AddPiece(ref string, size uint64, r io.Reader) (sectorID uint64, err error) { +func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealID uint64) (sectorID uint64, err error) { sectorID, err = s.sb.AddPiece(ref, size, r) - if err != nil { return 0, err } - s.lk.Lock() + s.waitingLk.Lock() _, exists := s.waiting[sectorID] if !exists { // pieces can share sectors s.waiting[sectorID] = make(chan struct{}) } - s.lk.Unlock() + s.waitingLk.Unlock() + + s.dealsLk.Lock() + defer s.dealsLk.Unlock() + + k := datastore.NewKey(fmt.Sprint(sectorID)) + e, err := s.deals.Get(k) + var deals dealMapping + switch err { + case nil: + if err := cbor.DecodeInto(e, &deals); err != nil { + return 0, err + } + if deals.Committed { + return 0, xerrors.Errorf("sector %d already committed", sectorID) + } + fallthrough + case datastore.ErrNotFound: + deals.DealIDs = append(deals.DealIDs, dealID) + d, err := cbor.DumpObject(&deals) + if err != nil { + return 0, err + } + if err := s.deals.Put(k, d); err != nil { + return 0, err + } + default: + return 0, err + } return sectorID, nil } +func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { + s.dealsLk.Lock() + defer s.dealsLk.Unlock() + + k := datastore.NewKey(fmt.Sprint(sectorID)) + e, err := s.deals.Get(k) + + switch err { + case nil: + var deals dealMapping + if err := cbor.DecodeInto(e, &deals); err != nil { + return nil, err + } + if deals.Committed { + log.Errorf("getting deal IDs for sector %d: sector already marked as committed", sectorID) + } + + deals.Committed = true + d, err := cbor.DumpObject(&deals) + if err != nil { + return nil, err + } + if err := s.deals.Put(k, d); err != nil { + return nil, err + } + + return deals.DealIDs, nil + case datastore.ErrNotFound: + log.Errorf("getting deal IDs for sector %d failed: %s", err) + return []uint64{}, nil + default: + return nil, err + } +} + func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { - s.lk.Lock() + s.waitingLk.Lock() var at = -1 for i, ch := range s.incoming { if ch == c { @@ -126,7 +209,7 @@ func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { } } if at == -1 { - s.lk.Unlock() + s.waitingLk.Unlock() return } if len(s.incoming) > 1 { @@ -135,21 +218,21 @@ func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { s.incoming[last] = nil } s.incoming = s.incoming[:len(s.incoming)-1] - s.lk.Unlock() + s.waitingLk.Unlock() } func (s *Store) Incoming() <-chan sectorbuilder.SectorSealingStatus { ch := make(chan sectorbuilder.SectorSealingStatus, 8) - s.lk.Lock() + s.waitingLk.Lock() s.incoming = append(s.incoming, ch) - s.lk.Unlock() + s.waitingLk.Unlock() return ch } func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) { - s.lk.Lock() + s.waitingLk.Lock() watch, ok := s.waiting[sector] - s.lk.Unlock() + s.waitingLk.Unlock() if ok { select { case <-watch: diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 413cd9d3c..28b864e7a 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -153,7 +153,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { } } -func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast uint64) (sectorID uint64, err error) { +func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) { size, err := r.Size() if err != nil { return 0, err @@ -166,7 +166,7 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, keepAtLeast intermediate: st.intermediate, } - return st.Store.AddPiece(refst.pieceRef, uint64(size), refst) + return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { From e148f330082ea2c615e4790c8c36247fb7974609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 17:12:50 +0200 Subject: [PATCH 028/230] on chain deals: Address review comments --- chain/actors/actor_storagemarket.go | 94 ++++++++++++++--------------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 2bd982bd7..0925259ad 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -7,7 +7,6 @@ import ( "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" - cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" @@ -58,10 +57,10 @@ type StorageMarketState struct { Balances cid.Cid // hamt Deals cid.Cid // amt - NextDealID uint64 // TODO: amt.LastIndex() + NextDealID uint64 // TODO: spec } -// TODO: serialization mode spec +// TODO: Drop in favour of car storage type SerializationMode = uint64 const ( @@ -293,9 +292,6 @@ type PublishStorageDealResponse struct { DealIDs []uint64 } -// TODO: spec says 'call by CommitSector in StorageMiningSubsystem', and then -// says that this should be called before CommitSector. For now assuming that -// they meant 2 separate methods, See 'ActivateStorageDeals' below func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.VMContext, params *PublishStorageDealsParams) ([]byte, ActorError) { var self StorageMarketState old := vmctx.Storage().GetHead() @@ -310,12 +306,27 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. // todo: handle duplicate deals + if len(params.Deals) == 0 { + return nil, aerrors.New(1, "no storage deals in params.Deals") + } + out := PublishStorageDealResponse{ DealIDs: make([]uint64, len(params.Deals)), } + workerBytes, aerr := vmctx.Send(params.Deals[0].Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + if aerr != nil { + return nil, aerr + } + providerWorker, err := address.NewFromBytes(workerBytes) + if err != nil { + return nil, aerrors.HandleExternalError(err, "parsing provider worker address bytes") + } + + // TODO: REVIEW: Do we want to check if provider exists in the power actor? + for i, deal := range params.Deals { - if err := self.validateDeal(vmctx, deal); err != nil { + if err := self.validateDeal(vmctx, deal, providerWorker); err != nil { return nil, err } @@ -340,7 +351,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return nil, aerrors.HandleExternalError(err, "storing state failed") } - aerr := vmctx.Storage().Commit(old, nroot) + aerr = vmctx.Storage().Commit(old, nroot) if aerr != nil { return nil, aerr } @@ -353,9 +364,8 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return outBuf.Bytes(), nil } -func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal) aerrors.ActorError { - // REVIEW: just > ? - if vmctx.BlockHeight() >= deal.Proposal.ProposalExpiration { +func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError { + if vmctx.BlockHeight() > deal.Proposal.ProposalExpiration { return aerrors.New(1, "deal proposal already expired") } @@ -363,16 +373,7 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage return aerrors.Absorb(err, 2, "verifying proposer signature") } - workerBytes, aerr := vmctx.Send(deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) - if aerr != nil { - return aerr - } - providerWorker, err := address.NewFromBytes(workerBytes) - if err != nil { - return aerrors.HandleExternalError(err, "parsing provider worker address bytes") - } - - err = deal.Verify(providerWorker) + err := deal.Verify(providerWorker) if err != nil { return aerrors.Absorb(err, 2, "verifying provider signature") } @@ -382,8 +383,6 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage return aerrors.New(4, "message not sent by deal participant") } - // TODO: REVIEW: Do we want to check if provider exists in the power actor? - // TODO: do some caching (changes gas so needs to be in spec too) b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, deal.Proposal.Client, providerWorker) if aerr != nil { @@ -500,30 +499,34 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.HandleExternalError(err, "loading deals amt") } + // TODO: Would be nice if send could assert actor type + workerBytes, aerr := vmctx.Send(vmctx.Message().From, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + if aerr != nil { + return nil, aerr + } + providerWorker, err := address.NewFromBytes(workerBytes) + if err != nil { + return nil, aerrors.HandleExternalError(err, "parsing provider worker address bytes") + } + for _, deal := range params.DealIDs { var dealInfo OnChainDeal if err := deals.Get(deal, &dealInfo); err != nil { - return nil, aerrors.HandleExternalError(err, "getting del info failed") + return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - encoded, err := CreateExecParams(StorageMinerCodeCid, &IsMinerParam{ - Addr: vmctx.Message().From, - }) - if err != nil { - return nil, err + if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + return nil, aerrors.New(1, "ProcessStorageDealsPayment can only be called by deal provider") } - ret, err := vmctx.Send(StoragePowerAddress, SPAMethods.IsMiner, types.NewInt(0), encoded) - if err != nil { - return nil, err + if vmctx.BlockHeight() < dealInfo.ActivationEpoch { + // TODO: This is probably fatal + return nil, aerrors.New(1, "ActivationEpoch lower than block height") } - if bytes.Equal(ret, cbg.CborBoolTrue) { - return nil, aerrors.New(1, "ProcessStorageDealsPayment can only be called by storage miner actors") - } - - if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { - // TODO: ??? + if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Deal.Proposal.Duration { + // Deal expired, miner should drop it + // TODO: process payment for the remainder of last proving period return nil, nil } @@ -531,17 +534,10 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // TODO: division is hard, this more than likely has some off-by-one issue toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealInfo.Deal.Proposal.Duration)) - // TODO: cache somehow to conserve gas - workerBytes, err := vmctx.Send(dealInfo.Deal.Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) - if err != nil { - return nil, err + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) + if aerr != nil { + return nil, aerr } - providerWorker, eerr := address.NewFromBytes(workerBytes) - if eerr != nil { - return nil, aerrors.HandleExternalError(eerr, "parsing provider worker address bytes") - } - - b, bnd, err := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) clientBal := b[0] providerBal := b[1] @@ -564,7 +560,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.HandleExternalError(err, "storing state failed") } - aerr := vmctx.Storage().Commit(old, nroot) + aerr = vmctx.Storage().Commit(old, nroot) if aerr != nil { return nil, aerr } From 9bd086bf7beb3a90abbda81a144ed67a764495f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 17:38:34 +0200 Subject: [PATCH 029/230] Use shorter code cids --- chain/actors/actors.go | 14 +++++++------- lotuspond/front/src/chain/code.js | 10 +++++++--- lotuspond/front/src/chain/methods.js | 16 +++++++++------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/chain/actors/actors.go b/chain/actors/actors.go index 032c6e573..10181eacd 100644 --- a/chain/actors/actors.go +++ b/chain/actors/actors.go @@ -39,11 +39,11 @@ func init() { return c } - AccountCodeCid = mustSum("filecoin/1.0/AccountActor") - StoragePowerCodeCid = mustSum("filecoin/1.0/StoragePowerActor") - StorageMarketCodeCid = mustSum("filecoin/1.0/StorageMarketActor") - StorageMinerCodeCid = mustSum("filecoin/1.0/StorageMinerActor") - MultisigCodeCid = mustSum("filecoin/1.0/MultisigActor") - InitCodeCid = mustSum("filecoin/1.0/InitActor") - PaymentChannelCodeCid = mustSum("filecoin/1.0/PaymentChannelActor") + AccountCodeCid = mustSum("fil/1/account") // TODO: spec + StoragePowerCodeCid = mustSum("fil/1/power") + StorageMarketCodeCid = mustSum("fil/1/market") + StorageMinerCodeCid = mustSum("fil/1/miner") + MultisigCodeCid = mustSum("fil/1/multisig") + InitCodeCid = mustSum("fil/1/init") + PaymentChannelCodeCid = mustSum("fil/1/paych") } diff --git a/lotuspond/front/src/chain/code.js b/lotuspond/front/src/chain/code.js index 8dd41e0d3..c8ffc0b03 100644 --- a/lotuspond/front/src/chain/code.js +++ b/lotuspond/front/src/chain/code.js @@ -1,5 +1,9 @@ export default { - init: 'filecoin/1.0/InitActor', - power: 'filecoin/1.0/StoragePowerActor', - market: 'filecoin/1.0/StorageMarketActor' + account: "fil/1/account", + power: "fil/1/power", + market: "fil/1/market", + miner: "fil/1/miner", + multisig: "fil/1/multisig", + init: "fil/1/init", + paych: "fil/1/paych", } diff --git a/lotuspond/front/src/chain/methods.js b/lotuspond/front/src/chain/methods.js index 426f7a960..b9e153ef7 100644 --- a/lotuspond/front/src/chain/methods.js +++ b/lotuspond/front/src/chain/methods.js @@ -1,10 +1,12 @@ +import code from "./code"; + export default { - "filecoin/1.0/AccountActor": [ + [code.account]: [ "Send", "Constructor", "GetAddress", ], - "filecoin/1.0/StoragePowerActor": [ + [code.power]: [ "Send", "Constructor", "CreateStorageMiner", @@ -15,7 +17,7 @@ export default { "IsMiner", "StorageCollateralForSize" ], - "filecoin/1.0/StorageMarketActor": [ + [code.market]: [ "Send", "Constructor", "WithdrawBalance", @@ -29,7 +31,7 @@ export default { "GetLastExpirationFromDealIDs", "ActivateStorageDeals", ], - "filecoin/1.0/StorageMinerActor": [ + [code.miner]: [ "Send", "Constructor", "CommitSector", @@ -50,7 +52,7 @@ export default { "PaymentVerifyInclusion", "PaymentVerifySector", ], - "filecoin/1.0/MultisigActor": [ + [code.multisig]: [ "Send", "Constructor", "Propose", @@ -62,13 +64,13 @@ export default { "SwapSigner", "ChangeRequirement", ], - "filecoin/1.0/InitActor": [ + [code.init]: [ "Send", "Constructor", "Exec", "GetIdForAddress" ], - "filecoin/1.0/PaymentChannelActor": [ + [code.paych]: [ "Send", "Constructor", "UpdateChannelState", From 0b1427e487187f6ca6ea1c505c4255b36891de25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 17:46:21 +0200 Subject: [PATCH 030/230] on chain deals: Fix lint warnings --- chain/actors/actor_storagemarket.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 0925259ad..1d06feb06 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -235,7 +235,8 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { for addr, b := range set { - if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &b); err != nil { + balance := b // to stop linter complaining + if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &balance); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") } } @@ -364,7 +365,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return outBuf.Bytes(), nil } -func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError { +func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError { if vmctx.BlockHeight() > deal.Proposal.ProposalExpiration { return aerrors.New(1, "deal proposal already expired") } @@ -384,7 +385,7 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage } // TODO: do some caching (changes gas so needs to be in spec too) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, deal.Proposal.Client, providerWorker) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Proposal.Client, providerWorker) if aerr != nil { return aerrors.Wrap(aerr, "getting client, and provider balances") } @@ -414,7 +415,7 @@ func (self *StorageMarketState) validateDeal(vmctx types.VMContext, deal Storage return aerr } - self.Balances = bcid + st.Balances = bcid return nil } From f2550f9e360ccbe015a346800b8ba476edfb2ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 24 Oct 2019 17:56:32 +0200 Subject: [PATCH 031/230] on chain deals: Deal not found is not fatal --- chain/actors/actor_storagemarket.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 1d06feb06..57f7171a5 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -440,7 +440,10 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types for _, deal := range params.Deals { var dealInfo OnChainDeal if err := deals.Get(deal, &dealInfo); err != nil { - return nil, aerrors.HandleExternalError(err, "getting del info failed") + if _, is := err.(*amt.ErrNotFound); is { + return nil, aerrors.New(3, "deal not found") + } + return nil, aerrors.HandleExternalError(err, "getting deal info failed") } if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { @@ -513,16 +516,19 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx for _, deal := range params.DealIDs { var dealInfo OnChainDeal if err := deals.Get(deal, &dealInfo); err != nil { + if _, is := err.(*amt.ErrNotFound); is { + return nil, aerrors.New(2, "deal not found") + } return nil, aerrors.HandleExternalError(err, "getting deal info failed") } if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { - return nil, aerrors.New(1, "ProcessStorageDealsPayment can only be called by deal provider") + return nil, aerrors.New(3, "ProcessStorageDealsPayment can only be called by deal provider") } if vmctx.BlockHeight() < dealInfo.ActivationEpoch { // TODO: This is probably fatal - return nil, aerrors.New(1, "ActivationEpoch lower than block height") + return nil, aerrors.New(4, "ActivationEpoch lower than block height") } if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Deal.Proposal.Duration { From 3137ffb347418dcb8cf6380aad6c3b9c5a0ec768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 25 Oct 2019 14:44:34 +0200 Subject: [PATCH 032/230] on chain deals: Check StoragePrice --- chain/deals/provider_states.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index f0261180e..e26775439 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -85,7 +85,17 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.Errorf("deal proposal already expired") } - // TODO: check StorageCollateral / StoragePrice + // TODO: check StorageCollateral + + // TODO: + minPrice := types.BigMul(p.ask.Ask.Price, types.BigMul(types.NewInt(deal.Proposal.Duration), types.NewInt(deal.Proposal.PieceSize))) + if deal.Proposal.StoragePrice.LessThan(minPrice) { + return nil, xerrors.Errorf("storage price less than asking price: %s < %s", deal.Proposal.StoragePrice, minPrice) + } + + if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize { + return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize) + } // check market funds clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil) From 1094e9aff918005d55206c854ce0f866e271670c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 25 Oct 2019 14:44:53 +0200 Subject: [PATCH 033/230] on chain deals: Better client window in pond --- api/api.go | 3 +++ lotuspond/front/src/Address.js | 16 +++--------- lotuspond/front/src/Client.js | 47 ++++++++++++++++++++++++---------- lotuspond/front/src/Fil.js | 22 ++++++++++++++++ lotuspond/front/src/index.css | 5 ++++ node/impl/common.go | 2 ++ 6 files changed, 68 insertions(+), 27 deletions(-) create mode 100644 lotuspond/front/src/Fil.js diff --git a/api/api.go b/api/api.go index e0e7c142e..be5b587fe 100644 --- a/api/api.go +++ b/api/api.go @@ -183,6 +183,9 @@ type Version struct { APIVersion uint32 // TODO: git commit / os / genesis cid? + + // Seconds + BlockDelay uint64 } func (v Version) String() string { diff --git a/lotuspond/front/src/Address.js b/lotuspond/front/src/Address.js index 71eae0979..db6870e9a 100644 --- a/lotuspond/front/src/Address.js +++ b/lotuspond/front/src/Address.js @@ -4,6 +4,7 @@ import ReactTooltip from 'react-tooltip' import * as multihash from "multihashes" import State from "./State" import methods from "./chain/methods" +import Fil from "./Fil"; function truncAddr(addr, len) { if (!addr) { @@ -15,17 +16,6 @@ function truncAddr(addr, len) { return addr } -function filStr(raw) { - if(typeof raw !== 'string') { - return raw - } - if(raw.length < 18) { - raw = '0'.repeat(18 - raw.length) - } - let out = (raw.substring(0, raw.length - 18) + '.' + raw.substring(raw.length - 18, raw.length)).replace(/\.0+|0+$/g, ''); - return out ? out : '0' -} - let sheet = document.createElement('style') document.body.appendChild(sheet); @@ -132,12 +122,12 @@ class Address extends React.Component { nonce =  Nc:{this.state.nonce}{nonce} } - let balance = : {filStr(this.state.balance)}  + let balance = : {{this.state.balance}}  if(this.props.nobalance) { balance = } if(this.props.short) { - actInfo = {actInfo}: {filStr(this.state.balance)} + actInfo = {actInfo}: {this.state.balance} balance = } diff --git a/lotuspond/front/src/Client.js b/lotuspond/front/src/Client.js index 1c669d193..35aaab148 100644 --- a/lotuspond/front/src/Client.js +++ b/lotuspond/front/src/Client.js @@ -1,6 +1,7 @@ import React from 'react'; import Address from "./Address"; import Window from "./Window"; +import Fil from "./Fil"; const dealStates = [ "Unknown", @@ -19,31 +20,43 @@ class Client extends React.Component { super(props) this.state = { + miners: ["t0101"], + ask: {Price: "3"}, + kbs: 1, blocks: 12, total: 36000, miner: "t0101", - deals: [] + deals: [], + + blockDelay: 10, } } - componentDidMount() { + async componentDidMount() { + let ver = await this.props.client.call('Filecoin.Version', []) + this.setState({blockDelay: ver.BlockDelay}) + this.getDeals() setInterval(this.getDeals, 1325) } getDeals = async () => { + let miners = await this.props.client.call('Filecoin.StateListMiners', [null]) let deals = await this.props.client.call('Filecoin.ClientListDeals', []) - this.setState({deals}) + miners.sort() + this.setState({deals, miners}) } update = (name) => (e) => this.setState({ [name]: e.target.value }); makeDeal = async () => { + let perBlk = this.state.ask.Price * this.state.kbs * 1000 + let file = await this.props.pondClient.call('Pond.CreateRandomFile', [this.state.kbs * 1000]) // 1024 won't fit in 1k blocks :( let cid = await this.props.client.call('Filecoin.ClientImport', [file]) - let dealcid = await this.props.client.call('Filecoin.ClientStartDeal', [cid, this.state.miner, `${Math.round(this.state.total / this.state.blocks)}`, Number(this.state.blocks)]) + let dealcid = await this.props.client.call('Filecoin.ClientStartDeal', [cid, this.state.miner, `${Math.round(perBlk)}`, Number(this.state.blocks)]) console.log("deal cid: ", dealcid) } @@ -65,17 +78,23 @@ class Client extends React.Component { } render() { - let ppb = Math.round(this.state.total / this.state.blocks * 100) / 100 - let ppmbb = Math.round(ppb / (this.state.kbs / 1000) * 100) / 100 + let perBlk = this.state.ask.Price * this.state.kbs * 1000 + let total = perBlk * this.state.blocks + let days = (this.state.blocks * this.state.blockDelay) / 60 / 60 / 24 let dealMaker = @@ -92,7 +111,7 @@ class Client extends React.Component { ) - return + return
{dealMaker}
{deals}
diff --git a/lotuspond/front/src/Fil.js b/lotuspond/front/src/Fil.js new file mode 100644 index 000000000..85b26e9ec --- /dev/null +++ b/lotuspond/front/src/Fil.js @@ -0,0 +1,22 @@ +import React from "react"; + +function filStr(raw) { + if(typeof raw !== 'string') { + raw = String(raw) + } + if(raw.length < 19) { + raw = '0'.repeat(19 - raw.length).concat(raw) + } + + let out = raw.substring(0, raw.length - 18).concat('.', raw.substring(raw.length - 18, raw.length)).replace(/\.0+$|0+$/g, ''); + return out ? out : '0' +} + + +class Fil extends React.Component { + render() { + return filStr(this.props.children) + } +} + +export default Fil \ No newline at end of file diff --git a/lotuspond/front/src/index.css b/lotuspond/front/src/index.css index 4a1df4db7..82ecd1213 100644 --- a/lotuspond/front/src/index.css +++ b/lotuspond/front/src/index.css @@ -11,3 +11,8 @@ code { font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace; } + +input[type=text] { + -webkit-appearance: none; + appearance: none; +} diff --git a/node/impl/common.go b/node/impl/common.go index 74f614ee3..f8037ca1d 100644 --- a/node/impl/common.go +++ b/node/impl/common.go @@ -87,6 +87,8 @@ func (a *CommonAPI) Version(context.Context) (api.Version, error) { return api.Version{ Version: build.Version, APIVersion: build.APIVersion, + + BlockDelay: build.BlockDelay, }, nil } From 4890f577f5a058dfe043caa3eae24bdc0a73f49f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 16 Oct 2019 16:07:16 +0900 Subject: [PATCH 034/230] give miners the ability to select different sector sizes --- api/api.go | 1 + api/struct.go | 6 +++ build/paramfetch.go | 2 +- build/params.go | 18 +++++++- chain/actors/actor_miner.go | 16 +++---- chain/actors/actor_storagepower.go | 11 +---- chain/actors/actor_storagepower_test.go | 2 +- chain/actors/actors_test.go | 2 +- chain/actors/cbor_gen.go | 57 +++++++++++++------------ chain/deals/client_states.go | 12 ++++-- chain/gen/gen.go | 14 ++++-- chain/gen/utils.go | 2 +- chain/stmgr/utils.go | 18 ++++++++ cli/createminer.go | 2 +- cmd/lotus-storage-miner/init.go | 9 +++- node/impl/full/state.go | 4 ++ node/impl/storminer.go | 9 +++- node/modules/storageminer.go | 12 ++++-- storage/miner.go | 14 +++++- 19 files changed, 144 insertions(+), 67 deletions(-) diff --git a/api/api.go b/api/api.go index be5b587fe..77edad959 100644 --- a/api/api.go +++ b/api/api.go @@ -129,6 +129,7 @@ type FullNode interface { StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) + StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) StatePledgeCollateral(context.Context, *types.TipSet) (types.BigInt, error) StateWaitMsg(context.Context, cid.Cid) (*MsgWait, error) StateListMiners(context.Context, *types.TipSet) ([]address.Address, error) diff --git a/api/struct.go b/api/struct.go index c76af4ed0..ce4bba8ef 100644 --- a/api/struct.go +++ b/api/struct.go @@ -92,6 +92,7 @@ type FullNodeStruct struct { StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` StateMinerProvingPeriodEnd func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` + StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"` StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"` StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` @@ -357,10 +358,15 @@ func (c *FullNodeStruct) StateMinerWorker(ctx context.Context, m address.Address func (c *FullNodeStruct) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) { return c.Internal.StateMinerPeerID(ctx, m, ts) } + func (c *FullNodeStruct) StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) { return c.Internal.StateMinerProvingPeriodEnd(ctx, actor, ts) } +func (c *FullNodeStruct) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) { + return c.Internal.StateMinerSectorSize(ctx, actor, ts) +} + func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) { return c.Internal.StateCall(ctx, msg, ts) } diff --git a/build/paramfetch.go b/build/paramfetch.go index 7816a2cca..25a030009 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -39,7 +39,7 @@ func GetParams(storage bool) error { } for name, info := range params { - if info.SectorSize != SectorSize { + if !SupportedSectorSize(info.SectorSize) { continue } if !storage && strings.HasSuffix(name, ".params") { diff --git a/build/params.go b/build/params.go index 780f7d931..bd7a6e29a 100644 --- a/build/params.go +++ b/build/params.go @@ -1,6 +1,8 @@ package build -import "math/big" +import ( + "math/big" +) // Core network constants @@ -10,7 +12,19 @@ import "math/big" const UnixfsChunkSize uint64 = 1 << 20 const UnixfsLinksPerLevel = 1024 -const SectorSize = 16 << 20 +var SectorSizes = []uint64{ + 16 << 20, + 1 << 30, +} + +func SupportedSectorSize(ssize uint64) bool { + for _, ss := range SectorSizes { + if ssize == ss { + return true + } + } + return false +} // ///// // Payments diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index fba8f94ad..646ce4d02 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -87,13 +87,13 @@ type MinerInfo struct { PeerID peer.ID // Amount of space in each sector committed to the network by this miner. - SectorSize types.BigInt + SectorSize uint64 } type StorageMinerConstructorParams struct { Owner address.Address Worker address.Address - SectorSize types.BigInt + SectorSize uint64 PeerID peer.ID } @@ -241,7 +241,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex } // Power of the miner after adding this sector - futurePower := types.BigAdd(self.Power, mi.SectorSize) + futurePower := types.BigAdd(self.Power, types.NewInt(mi.SectorSize)) collateralRequired := CollateralForPower(futurePower) // TODO: grab from market? @@ -399,7 +399,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext, faults := self.CurrentFaultSet.All() - if ok, lerr := sectorbuilder.VerifyPost(mi.SectorSize.Uint64(), + if ok, lerr := sectorbuilder.VerifyPost(mi.SectorSize, sectorbuilder.NewSortedSectorInfo(sectorInfos), seed, params.Proof, faults); !ok || lerr != nil { if lerr != nil { @@ -432,7 +432,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext, oldPower := self.Power self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), - mi.SectorSize) + types.NewInt(mi.SectorSize)) enc, err := SerializeParams(&UpdateStorageParams{Delta: types.BigSub(self.Power, oldPower)}) if err != nil { @@ -517,8 +517,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *OnChainSealVerifyInfo) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof) +func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } @@ -632,7 +632,7 @@ func (sma StorageMinerActor) GetSectorSize(act *types.Actor, vmctx types.VMConte return nil, err } - return mi.SectorSize.Bytes(), nil + return types.NewInt(mi.SectorSize).Bytes(), nil } type PaymentVerifyParams struct { diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index a8e47ea2a..ea564bc62 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -53,12 +53,12 @@ type StoragePowerState struct { type CreateStorageMinerParams struct { Owner address.Address Worker address.Address - SectorSize types.BigInt + SectorSize uint64 PeerID peer.ID } func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) { - if !SupportedSectorSize(params.SectorSize) { + if !build.SupportedSectorSize(params.SectorSize) { return nil, aerrors.New(1, "Unsupported sector size") } @@ -116,13 +116,6 @@ func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VM return naddr.Bytes(), nil } -func SupportedSectorSize(ssize types.BigInt) bool { - if ssize.Uint64() == build.SectorSize { - return true - } - return false -} - type ArbitrateConsensusFaultParams struct { Block1 *types.BlockHeader Block2 *types.BlockHeader diff --git a/chain/actors/actor_storagepower_test.go b/chain/actors/actor_storagepower_test.go index 76af74c9d..b11edc71c 100644 --- a/chain/actors/actor_storagepower_test.go +++ b/chain/actors/actor_storagepower_test.go @@ -42,7 +42,7 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) { &CreateStorageMinerParams{ Owner: ownerAddr, Worker: workerAddr, - SectorSize: types.NewInt(build.SectorSize), + SectorSize: build.SectorSizes[0], PeerID: "fakepeerid", }) ApplyOK(t, ret) diff --git a/chain/actors/actors_test.go b/chain/actors/actors_test.go index 57d559cf8..9e0c1186a 100644 --- a/chain/actors/actors_test.go +++ b/chain/actors/actors_test.go @@ -118,7 +118,7 @@ func TestStorageMarketActorCreateMiner(t *testing.T) { params := &StorageMinerConstructorParams{ Owner: maddr, Worker: maddr, - SectorSize: types.NewInt(build.SectorSize), + SectorSize: build.SectorSizes[0], PeerID: "fakepeerid", } var err error diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 48e0ddabc..3b1e37a41 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -421,8 +421,8 @@ func (t *StorageMinerConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.SectorSize (types.BigInt) - if err := t.SectorSize.MarshalCBOR(w); err != nil { + // t.t.SectorSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { return err } @@ -469,15 +469,16 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (types.BigInt) - - { - - if err := t.SectorSize.UnmarshalCBOR(br); err != nil { - return err - } + // t.t.SectorSize (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = extra // t.t.PeerID (peer.ID) { @@ -702,8 +703,8 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.SectorSize (types.BigInt) - if err := t.SectorSize.MarshalCBOR(w); err != nil { + // t.t.SectorSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { return err } return nil @@ -752,15 +753,16 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { t.PeerID = peer.ID(sval) } - // t.t.SectorSize (types.BigInt) - - { - - if err := t.SectorSize.UnmarshalCBOR(br); err != nil { - return err - } + // t.t.SectorSize (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = extra return nil } @@ -2377,8 +2379,8 @@ func (t *CreateStorageMinerParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.SectorSize (types.BigInt) - if err := t.SectorSize.MarshalCBOR(w); err != nil { + // t.t.SectorSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { return err } @@ -2425,15 +2427,16 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (types.BigInt) - - { - - if err := t.SectorSize.UnmarshalCBOR(br); err != nil { - return err - } + // t.t.SectorSize (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = extra // t.t.PeerID (peer.ID) { diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index bf32f57d6..bb30e8440 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -2,11 +2,12 @@ package deals import ( "context" + + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" - - "golang.org/x/xerrors" ) type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error @@ -108,7 +109,12 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) error { log.Info("DEAL SEALED!") // TODO: want? - /*ok, err := sectorbuilder.VerifyPieceInclusionProof(build.SectorSize, deal.Proposal.PieceSize, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) + /*ssize, err := stmgr.GetMinerSectorSize(ctx, c.sm, nil, deal.Proposal.MinerAddress) + if err != nil { + return xerrors.Errorf("failed to get miner sector size: %w", err) + } + + ok, err := sectorbuilder.VerifyPieceInclusionProof(ssize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) if err != nil { return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 89d52bbce..6032dd61c 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -208,12 +208,12 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add worker, err := stmgr.GetMinerWorkerRaw(ctx, cg.sm, st, m) if err != nil { - return nil, nil, err + return nil, nil, xerrors.Errorf("get miner worker: %w", err) } vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, lastTicket.VRFProof) if err != nil { - return nil, nil, err + return nil, nil, xerrors.Errorf("compute VRF: %w", err) } tick := &types.Ticket{ @@ -252,7 +252,7 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad msgs, err := cg.getRandomMessages() if err != nil { - return nil, err + return nil, xerrors.Errorf("get random messages: %w", err) } for len(blks) == 0 { @@ -279,6 +279,12 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad } fts := store.NewFullTipSet(blks) + fmt.Println("Made a block: ", fts.TipSet().Cids()) + if len(fts.TipSet().Cids()) > 1 { + for _, b := range blks { + fmt.Printf("block %s: %#v\n", b.Cid(), b.Header) + } + } return &MinedTipSet{ TipSet: fts, @@ -395,7 +401,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*ty func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, miner address.Address, a MiningCheckAPI) (bool, types.ElectionProof, error) { r, err := a.ChainGetRandomness(ctx, ts, ticks, build.RandomnessLookback) if err != nil { - return false, nil, err + return false, nil, xerrors.Errorf("chain get randomness: %w", err) } mworker, err := a.StateMinerWorker(ctx, miner, ts) diff --git a/chain/gen/utils.go b/chain/gen/utils.go index a01b0f20a..436160da0 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -239,7 +239,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid params := mustEnc(&actors.CreateStorageMinerParams{ Owner: owner, Worker: worker, - SectorSize: types.NewInt(build.SectorSize), + SectorSize: build.SectorSizes[0], PeerID: pid, }) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index af3787766..1dff70bbd 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -2,6 +2,7 @@ package stmgr import ( "context" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" @@ -9,6 +10,7 @@ import ( amt "github.com/filecoin-project/go-amt-ipld" cid "github.com/ipfs/go-cid" + hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" @@ -175,6 +177,22 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.Sectors) } +func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) { + var mas actors.StorageMinerActorState + _, err := sm.LoadActorState(ctx, maddr, &mas, ts) + if err != nil { + return 0, xerrors.Errorf("failed to load miner actor state: %w", err) + } + + cst := hamt.CSTFromBstore(sm.cs.Blockstore()) + var minfo actors.MinerInfo + if err := cst.Get(ctx, mas.Info, &minfo); err != nil { + return 0, xerrors.Errorf("failed to read miner info: %w", err) + } + + return minfo.SectorSize, nil +} + func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) { blks := amt.WrapBlockstore(bs) a, err := amt.LoadAMT(blks, ssc) diff --git a/cli/createminer.go b/cli/createminer.go index 25640b5fa..4247fe48e 100644 --- a/cli/createminer.go +++ b/cli/createminer.go @@ -53,7 +53,7 @@ var createMinerCmd = &cli.Command{ createMinerArgs := actors.CreateStorageMinerParams{ Worker: worker, Owner: owner, - SectorSize: types.NewInt(ssize), + SectorSize: ssize, PeerID: pid, } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index d4e7a7235..54d488a1b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -49,6 +49,11 @@ var initCmd = &cli.Command{ Aliases: []string{"o"}, Usage: "owner key to use", }, + &cli.Uint64Flag{ + Name: "sector-size", + Usage: "specify sector size to use", + Value: build.SectorSizes[0], + }, }, Action: func(cctx *cli.Context) error { log.Info("Initializing lotus storage miner") @@ -276,6 +281,8 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, c return address.Undef, err } + ssize := cctx.Uint64("sector-size") + worker := owner if cctx.String("worker") != "" { worker, err = address.NewFromString(cctx.String("worker")) @@ -295,7 +302,7 @@ func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, c params, err := actors.SerializeParams(&actors.CreateStorageMinerParams{ Owner: owner, Worker: worker, - SectorSize: types.NewInt(build.SectorSize), + SectorSize: ssize, PeerID: peerid, }) if err != nil { diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 6cad3d648..45f2aa94e 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -69,6 +69,10 @@ func (a *StateAPI) StateMinerProvingPeriodEnd(ctx context.Context, actor address return stmgr.GetMinerProvingPeriodEnd(ctx, a.StateManager, ts, actor) } +func (a *StateAPI) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) { + return stmgr.GetMinerSectorSize(ctx, a.StateManager, ts, actor) +} + func (a *StateAPI) StatePledgeCollateral(ctx context.Context, ts *types.TipSet) (types.BigInt, error) { param, err := actors.SerializeParams(&actors.PledgeCollateralParams{Size: types.NewInt(0)}) if err != nil { diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 526dd9fbf..5631e8e09 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -7,12 +7,13 @@ import ( "math/rand" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" + + "golang.org/x/xerrors" ) type StorageMinerAPI struct { @@ -31,7 +32,11 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { - size := sectorbuilder.UserBytesForSectorSize(build.SectorSize) + ssize, err := sm.Miner.SectorSize(ctx) + if err != nil { + return 0, xerrors.Errorf("failed to get miner sector size: %w", err) + } + size := sectorbuilder.UserBytesForSectorSize(ssize) // TODO: create a deal name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 54f800a5f..4b7e13b94 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -16,7 +16,6 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/lib/sectorbuilder" @@ -38,13 +37,18 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { return address.NewFromBytes(maddrb) } -func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbuilder.SectorBuilderConfig, error) { - return func(ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilderConfig, error) { +func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { + return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } + ssize, err := api.StateMinerSectorSize(context.TODO(), minerAddr, nil) + if err != nil { + return nil, err + } + sp, err := homedir.Expand(storagePath) if err != nil { return nil, err @@ -56,7 +60,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS) (*sectorbui sb := §orbuilder.SectorBuilderConfig{ Miner: minerAddr, - SectorSize: build.SectorSize, + SectorSize: ssize, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, diff --git a/storage/miner.go b/storage/miner.go index 659a09c59..d3992a127 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -12,7 +12,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/events" @@ -55,6 +54,7 @@ type storageMinerApi interface { StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error) StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) + StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error) @@ -122,7 +122,12 @@ func (m *Miner) handlePostingSealedSectors(ctx context.Context) { func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { log.Info("committing sector") - ok, err := sectorbuilder.VerifySeal(build.SectorSize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) + ssize, err := m.SectorSize(ctx) + if err != nil { + return xerrors.Errorf("failed to check out own sector size: %w", err) + } + + ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) if err != nil { log.Error("failed to verify seal we just created: ", err) } @@ -200,3 +205,8 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { log.Infof("starting up miner %s, worker addr %s", m.maddr, m.worker) return nil } + +func (m *Miner) SectorSize(ctx context.Context) (uint64, error) { + // TODO: cache this + return m.api.StateMinerSectorSize(ctx, m.maddr, nil) +} From 5bed18de489671bdc3c77ea28b3cbf1b8ac58938 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 16 Oct 2019 16:09:08 +0900 Subject: [PATCH 035/230] add 256MB sectors --- build/params.go | 1 + 1 file changed, 1 insertion(+) diff --git a/build/params.go b/build/params.go index bd7a6e29a..9185a5b0c 100644 --- a/build/params.go +++ b/build/params.go @@ -14,6 +14,7 @@ const UnixfsLinksPerLevel = 1024 var SectorSizes = []uint64{ 16 << 20, + 256 << 20, 1 << 30, } From d753c39133573b5600ff9e882faeb534d113ad55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Oct 2019 06:11:47 +0200 Subject: [PATCH 036/230] Add rate limits to the fountain by @travisperson --- cmd/lotus-fountain/main.go | 40 +++++++++++ cmd/lotus-fountain/rate_limiter.go | 95 +++++++++++++++++++++++++ cmd/lotus-fountain/rate_limiter_test.go | 38 ++++++++++ go.mod | 1 + go.sum | 1 + 5 files changed, 175 insertions(+) create mode 100644 cmd/lotus-fountain/rate_limiter.go create mode 100644 cmd/lotus-fountain/rate_limiter_test.go diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 1a7a87ce4..eccc4ac5c 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "time" rice "github.com/GeertJohan/go.rice" logging "github.com/ipfs/go-log" @@ -88,6 +89,22 @@ var runCmd = &cli.Command{ ctx: ctx, api: nodeApi, from: from, + limiter: NewLimiter(LimiterConfig{ + TotalRate: time.Second, + TotalBurst: 20, + IPRate: 5 * time.Minute, + IPBurst: 5, + WalletRate: time.Hour, + WalletBurst: 1, + }), + colLimiter: NewLimiter(LimiterConfig{ + TotalRate: time.Second, + TotalBurst: 20, + IPRate: 24 * time.Hour, + IPBurst: 1, + WalletRate: 24 * 364 * time.Hour, + WalletBurst: 1, + }), } http.Handle("/", http.FileServer(rice.MustFindBox("site").HTTPBox())) @@ -110,9 +127,25 @@ type handler struct { api api.FullNode from address.Address + + limiter *Limiter + colLimiter *Limiter } func (h *handler) send(w http.ResponseWriter, r *http.Request) { + // General limiter to allow throttling all messages that can make it into the mpool + if !h.limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + + // Limit based on IP + limiter := h.limiter.GetIPLimiter(r.RemoteAddr) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + to, err := address.NewFromString(r.FormValue("address")) if err != nil { w.WriteHeader(400) @@ -120,6 +153,13 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { return } + // Limit based on wallet address + limiter = h.limiter.GetWalletLimiter(to.String()) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ Value: sendPerRequest, From: h.from, diff --git a/cmd/lotus-fountain/rate_limiter.go b/cmd/lotus-fountain/rate_limiter.go new file mode 100644 index 000000000..5058049fc --- /dev/null +++ b/cmd/lotus-fountain/rate_limiter.go @@ -0,0 +1,95 @@ + +package main + +import ( + "sync" + "time" + + "golang.org/x/time/rate" +) + +type Limiter struct { + control *rate.Limiter + + ips map[string]*rate.Limiter + wallets map[string]*rate.Limiter + mu *sync.RWMutex + + config LimiterConfig +} + +type LimiterConfig struct { + TotalRate time.Duration + TotalBurst int + + IPRate time.Duration + IPBurst int + + WalletRate time.Duration + WalletBurst int +} + +func NewLimiter(c LimiterConfig) *Limiter { + return &Limiter{ + control: rate.NewLimiter(rate.Every(c.TotalRate), c.TotalBurst), + mu: &sync.RWMutex{}, + ips: make(map[string]*rate.Limiter), + wallets: make(map[string]*rate.Limiter), + + config: c, + } +} + +func (i *Limiter) Allow() bool { + return i.control.Allow() +} + +func (i *Limiter) AddIPLimiter(ip string) *rate.Limiter { + i.mu.Lock() + defer i.mu.Unlock() + + limiter := rate.NewLimiter(rate.Every(i.config.IPRate), i.config.IPBurst) + + i.ips[ip] = limiter + + return limiter +} + +func (i *Limiter) GetIPLimiter(ip string) *rate.Limiter { + i.mu.Lock() + limiter, exists := i.ips[ip] + + if !exists { + i.mu.Unlock() + return i.AddIPLimiter(ip) + } + + i.mu.Unlock() + + return limiter +} + +func (i *Limiter) AddWalletLimiter(addr string) *rate.Limiter { + i.mu.Lock() + defer i.mu.Unlock() + + limiter := rate.NewLimiter(rate.Every(i.config.WalletRate), i.config.WalletBurst) + + i.wallets[addr] = limiter + + return limiter +} + +func (i *Limiter) GetWalletLimiter(wallet string) *rate.Limiter { + i.mu.Lock() + limiter, exists := i.wallets[wallet] + + if !exists { + i.mu.Unlock() + return i.AddWalletLimiter(wallet) + } + + i.mu.Unlock() + + return limiter +} diff --git a/cmd/lotus-fountain/rate_limiter_test.go b/cmd/lotus-fountain/rate_limiter_test.go new file mode 100644 index 000000000..03590de50 --- /dev/null +++ b/cmd/lotus-fountain/rate_limiter_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRateLimit(t *testing.T) { + limiter := NewLimiter(LimiterConfig{ + TotalRate: time.Second, + TotalBurst: 20, + IPRate: time.Second, + IPBurst: 1, + WalletRate: time.Second, + WalletBurst: 1, + }) + + for i := 0; i < 20; i++ { + assert.True(t, limiter.Allow()) + } + + assert.False(t, limiter.Allow()) + + time.Sleep(time.Second) + assert.True(t, limiter.Allow()) + + assert.True(t, limiter.GetIPLimiter("127.0.0.1").Allow()) + assert.False(t, limiter.GetIPLimiter("127.0.0.1").Allow()) + time.Sleep(time.Second) + assert.True(t, limiter.GetIPLimiter("127.0.0.1").Allow()) + + assert.True(t, limiter.GetWalletLimiter("abc123").Allow()) + assert.False(t, limiter.GetWalletLimiter("abc123").Allow()) + time.Sleep(time.Second) + assert.True(t, limiter.GetWalletLimiter("abc123").Allow()) +} diff --git a/go.mod b/go.mod index 48098edda..d72aaa4cd 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect + golang.org/x/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 google.golang.org/api v0.9.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 diff --git a/go.sum b/go.sum index be3b0270d..677d05b9b 100644 --- a/go.sum +++ b/go.sum @@ -640,6 +640,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 43e1752816661d828c823399b047726866ff0f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Oct 2019 10:12:53 +0200 Subject: [PATCH 037/230] fountain: Create miner enpoint --- cmd/lotus-fountain/main.go | 132 ++++++++++++++++++++++++++++- cmd/lotus-fountain/rate_limiter.go | 1 - cmd/lotus-fountain/site/index.html | 4 +- 3 files changed, 133 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index eccc4ac5c..bee3e8b71 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -9,11 +9,13 @@ import ( rice "github.com/GeertJohan/go.rice" logging "github.com/ipfs/go-log" + peer "github.com/libp2p/go-libp2p-peer" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -178,6 +180,132 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { } func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(400) - // todo + // General limiter owner allow throttling all messages that can make it into the mpool + if !h.colLimiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + + // Limit based on IP + limiter := h.colLimiter.GetIPLimiter(r.RemoteAddr) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + + owner, err := address.NewFromString(r.FormValue("address")) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + if owner.Protocol() != address.BLS { + w.WriteHeader(400) + w.Write([]byte("Miner address must use BLS")) + return + } + + log.Infof("mkactoer on %s", owner) + + // Limit based on wallet address + limiter = h.colLimiter.GetWalletLimiter(owner.String()) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + collateral, err := h.api.StatePledgeCollateral(r.Context(), nil) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ + Value: sendPerRequest, + From: h.from, + To: owner, + + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000), + }) + if err != nil { + w.WriteHeader(400) + w.Write([]byte("pushfunds: " + err.Error())) + return + } + log.Infof("push funds to %s: %s", owner, smsg.Cid()) + + mw, err := h.api.StateWaitMsg(r.Context(), smsg.Cid()) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + if mw.Receipt.ExitCode != 0 { + w.WriteHeader(400) + w.Write([]byte(xerrors.Errorf("create storage miner failed: exit code %d", mw.Receipt.ExitCode).Error())) + return + } + + log.Infof("sendto %s ok", owner) + + params, err := actors.SerializeParams(&actors.CreateStorageMinerParams{ + Owner: owner, + Worker: owner, + SectorSize: build.SectorSizes[0], // TODO: dropdown allowing selection + PeerID: peer.ID("SETME"), + }) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + createStorageMinerMsg := &types.Message{ + To: actors.StorageMarketAddress, + From: h.from, + Value: collateral, + + Method: actors.SPAMethods.CreateStorageMiner, + Params: params, + + GasLimit: types.NewInt(10000000), + GasPrice: types.NewInt(0), + } + + signed, err := h.api.MpoolPushMessage(r.Context(), createStorageMinerMsg) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + log.Infof("smc %s", owner) + + mw, err = h.api.StateWaitMsg(r.Context(), signed.Cid()) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + if mw.Receipt.ExitCode != 0 { + w.WriteHeader(400) + w.Write([]byte(xerrors.Errorf("create storage miner failed: exit code %d", mw.Receipt.ExitCode).Error())) + return + } + + addr, err := address.NewFromBytes(mw.Receipt.Return) + if err != nil { + w.WriteHeader(400) + w.Write([]byte(err.Error())) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(200) + fmt.Fprintf(w, "New storage miners address is: %s\n", addr) + fmt.Fprintf(w, "Run lotus-storage-miner init --actor=%s -owner=%s", addr, owner) } diff --git a/cmd/lotus-fountain/rate_limiter.go b/cmd/lotus-fountain/rate_limiter.go index 5058049fc..eb7215780 100644 --- a/cmd/lotus-fountain/rate_limiter.go +++ b/cmd/lotus-fountain/rate_limiter.go @@ -1,4 +1,3 @@ - package main import ( diff --git a/cmd/lotus-fountain/site/index.html b/cmd/lotus-fountain/site/index.html index 0a3d3577b..632470b7f 100644 --- a/cmd/lotus-fountain/site/index.html +++ b/cmd/lotus-fountain/site/index.html @@ -14,7 +14,9 @@
Enter destination address: - + +
+When creating storage miner, DO NOT REFRESH THE PAGE, wait for it to load. This can take more than 5min. From a20255d60643ddb005401979ebe306fbe79974bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Oct 2019 16:28:03 +0200 Subject: [PATCH 038/230] Fancier faucet --- cmd/lotus-fountain/main.go | 12 +++---- cmd/lotus-fountain/site/funds.html | 29 ++++++++++++++++ cmd/lotus-fountain/site/index.html | 33 ++++++++++-------- cmd/lotus-fountain/site/main.css | 56 ++++++++++++++++++++++++++++++ cmd/lotus-fountain/site/miner.html | 43 +++++++++++++++++++++++ 5 files changed, 153 insertions(+), 20 deletions(-) create mode 100644 cmd/lotus-fountain/site/funds.html create mode 100644 cmd/lotus-fountain/site/main.css create mode 100644 cmd/lotus-fountain/site/miner.html diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index bee3e8b71..705c7d148 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -94,17 +94,17 @@ var runCmd = &cli.Command{ limiter: NewLimiter(LimiterConfig{ TotalRate: time.Second, TotalBurst: 20, - IPRate: 5 * time.Minute, + IPRate: time.Minute, IPBurst: 5, - WalletRate: time.Hour, + WalletRate: 15 * time.Minute, WalletBurst: 1, }), colLimiter: NewLimiter(LimiterConfig{ TotalRate: time.Second, TotalBurst: 20, - IPRate: 24 * time.Hour, + IPRate: 10 * time.Minute, IPBurst: 1, - WalletRate: 24 * 364 * time.Hour, + WalletRate: 1 * time.Hour, WalletBurst: 1, }), } @@ -206,7 +206,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { return } - log.Infof("mkactoer on %s", owner) + log.Infof("mkactor on %s", owner) // Limit based on wallet address limiter = h.colLimiter.GetWalletLimiter(owner.String()) @@ -307,5 +307,5 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.WriteHeader(200) fmt.Fprintf(w, "New storage miners address is: %s\n", addr) - fmt.Fprintf(w, "Run lotus-storage-miner init --actor=%s -owner=%s", addr, owner) + fmt.Fprintf(w, "Run lotus-storage-miner init --actor=%s --owner=%s", addr, owner) } diff --git a/cmd/lotus-fountain/site/funds.html b/cmd/lotus-fountain/site/funds.html new file mode 100644 index 000000000..cd26032f3 --- /dev/null +++ b/cmd/lotus-fountain/site/funds.html @@ -0,0 +1,29 @@ + + + + Sending Funds - Lotus Fountain + + + +
+
+
+ [SENDING FUNDS] +
+
+
+ Enter destination address: + + +
+
+
+
+
+ [Back] + Not dispensing real Filecoin tokens +
+
+
+ + diff --git a/cmd/lotus-fountain/site/index.html b/cmd/lotus-fountain/site/index.html index 632470b7f..85226e4c0 100644 --- a/cmd/lotus-fountain/site/index.html +++ b/cmd/lotus-fountain/site/index.html @@ -2,21 +2,26 @@ Lotus Fountain - + -
- Enter destination address: - - - -
-When creating storage miner, DO NOT REFRESH THE PAGE, wait for it to load. This can take more than 5min. +
+
+
+ [LOTUS DEVNET FAUCET] +
+
+ [Send Funds] +
+
+ [Create Miner] +
+
+
+
+ Not dispensing real Filecoin tokens +
+
+
diff --git a/cmd/lotus-fountain/site/main.css b/cmd/lotus-fountain/site/main.css new file mode 100644 index 000000000..efcd5dc26 --- /dev/null +++ b/cmd/lotus-fountain/site/main.css @@ -0,0 +1,56 @@ +body { + font-family: 'monospace'; + background: #1f1f1f; + color: #f0f0f0; + padding: 0; + margin: 0; +} + +.Index { + width: 100vw; + height: 100vh; + background: #1a1a1a; + color: #f0f0f0; + font-family: monospace; + + display: grid; + grid-template-columns: auto 40vw auto; + grid-template-rows: auto auto auto 3em; + grid-template-areas: + ". . ." + ". main ." + ". . ." + "footer footer footer"; +} +.Index-footer { + background: #2a2a2a; + grid-area: footer; +} + +.Index-footer > div { + padding-left: 0.7em; + padding-top: 0.7em; +} + +.Index-nodes { + grid-area: main; + background: #2a2a2a; +} + +.Index-node { + margin: 5px; + padding: 15px; + background: #1f1f1f; +} + +a:link { + color: #50f020; +} + +a:visited { + color: #50f020; +} + +a:hover { + color: #30a00a; +} diff --git a/cmd/lotus-fountain/site/miner.html b/cmd/lotus-fountain/site/miner.html new file mode 100644 index 000000000..31952269e --- /dev/null +++ b/cmd/lotus-fountain/site/miner.html @@ -0,0 +1,43 @@ + + + + Creating Storage Miner - Lotus Fountain + + + +
+
+
+ [CREATING STORAGE MINER] +
+
+
+ Enter destination address: + + +
+
+ +
+ When creating storage miner, DO NOT REFRESH THE PAGE, wait for it to load. This can take more than 5min. +
+
+
+
+ [Back] + Not dispensing real Filecoin tokens +
+
+
+ + + \ No newline at end of file From 874ab1456c1442544f87fb364554bd3b781d06ee Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 21 Oct 2019 19:58:41 +0800 Subject: [PATCH 039/230] WIP: updating to new proofs code --- chain/actors/actor_miner.go | 18 ++-- chain/actors/cbor_gen.go | 22 ++--- cmd/lotus-storage-miner/sectors.go | 3 +- extern/go-sectorbuilder | 2 +- go.sum | 131 +++++++++++++++++++++++++++++ lib/sectorbuilder/sectorbuilder.go | 16 ++-- storage/miner.go | 9 +- 7 files changed, 164 insertions(+), 37 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 5b4ab3e3f..dc5f8c530 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -200,11 +200,10 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ } type OnChainSealVerifyInfo struct { - CommD []byte // TODO: update proofs code - CommR []byte - CommRStar []byte + CommD []byte // TODO: update proofs code + CommR []byte - //Epoch uint64 + Epoch uint64 Proof []byte DealIDs []uint64 SectorNumber uint64 @@ -229,7 +228,12 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex // TODO: this needs to get normalized to either the ID address or the actor address maddr := vmctx.Message().To - if ok, err := ValidatePoRep(maddr, mi.SectorSize, params); err != nil { + ticket, err := vmctx.GetRandomness(params.Epoch) + if err != nil { + return nil, aerrors.Wrap(err, "failed to get randomness for commitsector") + } + + if ok, err := ValidatePoRep(maddr, mi.SectorSize, params, ticket); err != nil { return nil, err } else if !ok { return nil, aerrors.New(2, "bad proof!") @@ -521,8 +525,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, params.CommRStar, maddr, params.SectorNumber, params.Proof) +func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo, ticket []byte) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, maddr, ticket, params.SectorNumber, params.Proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 8a0231c43..b45586c96 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -517,11 +517,8 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.CommRStar ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommRStar)))); err != nil { - return err - } - if _, err := w.Write(t.CommRStar); err != nil { + // t.t.Epoch (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Epoch)); err != nil { return err } @@ -599,23 +596,16 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } - // t.t.CommRStar ([]uint8) + // t.t.Epoch (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommRStar: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommRStar = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommRStar); err != nil { - return err + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") } + t.Epoch = extra // t.t.Proof ([]uint8) maj, extra, err = cbg.CborReadHeader(br) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index deede68ad..dd7e0ac47 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -71,7 +71,8 @@ var sectorsStatusCmd = &cli.Command{ fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg) fmt.Printf("CommD:\t\t%x\n", status.CommD) fmt.Printf("CommR:\t\t%x\n", status.CommR) - fmt.Printf("CommR*:\t\t%x\n", status.CommRStar) + fmt.Printf("Ticket:\t\t%x\n", status.Ticket.TicketBytes) + fmt.Printf("TicketH:\t\t%d\n", status.Ticket.BlockHeight) fmt.Printf("Proof:\t\t%x\n", status.Proof) fmt.Printf("Pieces:\t\t%v\n", status.Pieces) return nil diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 32e85ba52..692725ff2 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 32e85ba52d9ccf3fc5715bcf53265b042020ee82 +Subproject commit 692725ff21919ce9c9df9ea87621b0c1e6a9746c diff --git a/go.sum b/go.sum index 677d05b9b..3d9bc15a9 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,10 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= @@ -28,6 +30,8 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -71,6 +75,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA= @@ -83,16 +88,35 @@ github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -102,11 +126,29 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= +github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -121,6 +163,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -131,6 +174,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -247,6 +291,8 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -254,8 +300,13 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -382,8 +433,10 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= +github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= @@ -392,6 +445,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -399,6 +453,7 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -410,9 +465,17 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -440,21 +503,30 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -469,16 +541,32 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -488,17 +576,27 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= +github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c= +github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -508,12 +606,19 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -571,11 +676,13 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -585,9 +692,11 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -601,6 +710,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -613,6 +723,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -625,6 +736,7 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -632,27 +744,38 @@ golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -673,20 +796,24 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -698,3 +825,7 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 242259e30..a9790e9ba 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -52,8 +52,8 @@ func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { }, nil } -func addressToProverID(a address.Address) [31]byte { - var proverId [31]byte +func addressToProverID(a address.Address) [32]byte { + var proverId [32]byte copy(proverId[:], a.Payload()) return proverId } @@ -88,7 +88,9 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, err } func (sb *SectorBuilder) SealAllStagedSectors() error { - return sectorbuilder.SealAllStagedSectors(sb.handle) + panic("dont call this") + _, err := sectorbuilder.SealAllStagedSectors(sb.handle, sectorbuilder.SealTicket{}) + return err } func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { @@ -121,14 +123,14 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector -func VerifySeal(sectorSize uint64, commR, commD, commRStar []byte, proverID address.Address, sectorID uint64, proof []byte) (bool, error) { - var commRa, commDa, commRStara [32]byte +func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, sectorID uint64, proof []byte) (bool, error) { + var commRa, commDa, ticketa [32]byte copy(commRa[:], commR) copy(commDa[:], commD) - copy(commRStara[:], commRStar) + copy(ticketa[:], ticket) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, commRStara, proverIDa, sectorID, proof) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, sectorID, proof) } func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP []byte, commD []byte, proof []byte) (bool, error) { diff --git a/storage/miner.go b/storage/miner.go index d3992a127..599634ddb 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -127,7 +127,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal return xerrors.Errorf("failed to check out own sector size: %w", err) } - ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], sinfo.CommRStar[:], m.maddr, sinfo.SectorID, sinfo.Proof) + ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], m.maddr, sinfo.Ticket.TicketBytes[:], sinfo.SectorID, sinfo.Proof) if err != nil { log.Error("failed to verify seal we just created: ", err) } @@ -141,10 +141,9 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal } params := &actors.OnChainSealVerifyInfo{ - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - CommRStar: sinfo.CommRStar[:], - Proof: sinfo.Proof, + CommD: sinfo.CommD[:], + CommR: sinfo.CommR[:], + Proof: sinfo.Proof, DealIDs: deals, SectorNumber: sinfo.SectorID, From 5257b1cce1e4006dad4e776abd8ae2e119a900be Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 22 Oct 2019 14:43:14 +0800 Subject: [PATCH 040/230] should use the new parameters --- build/paramfetch.go | 3 +- build/proof-params/parameters.json | 114 ++++++++++++------------ lib/sectorbuilder/sectorbuilder_test.go | 9 +- 3 files changed, 66 insertions(+), 60 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index 25a030009..bf3e5b719 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -17,7 +17,8 @@ import ( var log = logging.Logger("build") -const gateway = "http://198.211.99.118/ipfs/" +//const gateway = "http://198.211.99.118/ipfs/" +const gateway = "https://ipfs.io/ipfs/" const paramdir = "/var/tmp/filecoin-proof-parameters" type paramFile struct { diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json index 723570a3d..0460fb226 100644 --- a/build/proof-params/parameters.json +++ b/build/proof-params/parameters.json @@ -1,82 +1,82 @@ { - "v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { - "cid": "QmZ6Y88jRbRjjYQzhh8o85bcUeChj7NGyo9yK6VbywhJ9F", - "digest": "3d245479d9b5fc668d58c493da5f3ee1", + "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { + "cid": "QmT22f1Np1GpW29NXD7Zrv3Ae4poMYhmkDjyscqL8QrJXY", + "digest": "989fd8d989e0f7f1fe21bb010cf1b231", "sector_size": 16777216 }, - "v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { - "cid": "QmbGgBLMTCnRc1E1fsUCPyZE4SYzrtAYEWCqCtL55AgxE2", - "digest": "29bd4c152096f878f41257b433159d81", + "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { + "cid": "QmVqSdc23to4UwduCCb25223rpSccvtcgPMfRKY1qjucDc", + "digest": "c6d258c37243b8544238a98100e3e399", "sector_size": 16777216 }, - "v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { - "cid": "Qmdm8vhWeRsZUUaHdysDr91gv6u6RFeC18hHxGnnzPrwcW", - "digest": "c67fd415a65e6d1caf4278597cf3462e", + "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { + "cid": "QmRTCqgokEGTMfWVaSr7qFXTNotmpd2QBEi8RsvSQKmPLz", + "digest": "ff77a5e270afc6e1c7fbc19e48348fac", "sector_size": 1073741824 }, - "v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { - "cid": "Qmc3xssw1syaZDZKmiM7TbCRnoDPfmD6V8ec1eTESidJQS", - "digest": "870355c10000010b9a4ece80892acca2", + "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { + "cid": "QmRssVAXRN3xp9VdSpTq1pNjkob3QiikoFZiM5hqrmh1VU", + "digest": "b41f35ac26224258e366327716a835a4", "sector_size": 1073741824 }, - "v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { - "cid": "QmZBvF2F9wTYKLBxWSCQKe34D3M7vkNNc7ou8mxnNhZkZc", - "digest": "5d854e0ecfbd12cb7fa1247a6e6a0315", + "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { + "cid": "QmYNVRVzjXkuxJfnHTU5vmEcUBQf8dabXZ4m53SzqMkBv5", + "digest": "d156b685e4a1fe3a1f7230b6a39b5ad4", "sector_size": 1024 }, - "v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { - "cid": "QmZfdHrnk2oN3Gx7hhjpRGXu8qY6FcqLjpHQ6jt1BBDg5R", - "digest": "aca566faa466f05fb9d622bec39e4b6d", + "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { + "cid": "QmaCEcsCFVuepMKdC5WURbr5ucEyLMNGxQaB7HqSnr2KGh", + "digest": "06ff067ac78cdab5d7bbc82170882241", "sector_size": 1024 }, - "v12-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { - "cid": "QmYdGGwQXpaBGTVWXqMFVXUP2CZhtsV29jxPkRm54ArAdT", - "digest": "eb2d3c4cb7b32c87ead5326bcbd495f3", + "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { + "cid": "QmVuabRvJ797NwLisGKwRURASGxopBBgg4rfNsbZoSYzAc", + "digest": "0e1ceb79a459a60508f480e5b1fed7ac", "sector_size": 268435456 }, - "v12-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { - "cid": "QmeTtWQ2hCUq34BpHTy21jJqVqHbPJdNhQRqW4SF4ZNA7v", - "digest": "c83eca165ba94233861227578d658a22", + "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { + "cid": "QmdWENZBAbuUty1vVNn9vmvj1XbJ5UC8qzpcVD35s5AJxG", + "digest": "1b755c74b9d6823c014f6a7ef76249f2", "sector_size": 268435456 }, - "v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.params": { - "cid": "QmTkeyz3mfec4MqCbbiwuVWAQicY231zpdxSjdxN5U84PD", - "digest": "cf7118ac2273e2ccb6b451a5acd5f6e0", - "sector_size": 1073741824 - }, - "v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.vk": { - "cid": "QmaGzJCwJNpQAD63QAKb8y4MKS1AMHeKyYzg3C5WyRSRRM", - "digest": "288c792f4fe09c85f8f35673fb9d5ed0", - "sector_size": 1073741824 - }, - "v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.params": { - "cid": "QmNSuxq15JPFCTehxVpgJydNZ79rpLoNwnLzQMGA9EziXg", - "digest": "818cd9cc2e0e47210a05bd073847ab5a", - "sector_size": 268435456 - }, - "v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.vk": { - "cid": "Qmbc8LcydZXsVqQrkNMeLEu31Vxi1VigQGJ2ehytxWPALH", - "digest": "a6636e2ee1a176161e022296bc045e79", - "sector_size": 268435456 - }, - "v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.params": { - "cid": "QmUBpwbVwu5uzjTS8n1yesJ2QUaVbWJ8D9p2VaSsfjgAUr", - "digest": "e7aa73a1b06290d30f567bfac8324bf1", + "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": { + "cid": "QmPG38HmDNVFiQJskqKe9sfSjyHfvZRihcfry78rt22FDT", + "digest": "8ea0b47e72250d5d6dab5d4f859e65de", "sector_size": 16777216 }, - "v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.vk": { - "cid": "Qme2uK8sQJUT3DzF291p2b91eFQzn7cKSJFiKVBsvrjTgt", - "digest": "d318cd116803c8ccbd3ac4cded9400ad", + "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": { + "cid": "Qmd3pNM22pgAoRT24tNyEZmeEWK2GtoZznBvzjie2YgqCn", + "digest": "e39f344757c919ae6bbc9b61311c73b2", "sector_size": 16777216 }, - "v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.params": { - "cid": "QmTfhTnkFvbpFfw8UydFdnPCDfxgAxEcw4fRdGsELpcFnh", - "digest": "906b6c0c9dc5bb581d9641c11b54e197", + "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": { + "cid": "QmQsS6RqWmgdwPnHCwhBJH3WDPcAxhKfbQUs2bwa8D9su8", + "digest": "09879a69abcc51de5c1095f347c84e2b", + "sector_size": 268435456 + }, + "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": { + "cid": "QmSFBL5rg2TJv8QjLzrbr4c2KV2uDNN13RBVNUgwemJgM1", + "digest": "db0f245f7e9989879d2fa6328bd57d32", + "sector_size": 268435456 + }, + "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": { + "cid": "QmZ9UVBfviaNFsKyazA4k8GZcM1tHNDpDyrEK9qkxaMJXx", + "digest": "402a7c7c82eaa4af9fba3c7e4402b65b", + "sector_size": 1073741824 + }, + "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": { + "cid": "QmNmDcPVJ1bFFDcNCcRnEoQ6vNDNpKadLPHyEpUoF47gxV", + "digest": "2741c456346a3758e88249d1f4c0d227", + "sector_size": 1073741824 + }, + "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": { + "cid": "QmNqgCv6UjdKDNMuiwDweZ22TQYMd3gV6nWiA5PjMtNDVu", + "digest": "a9d316d0dbca152e653d41ad8e40058a", "sector_size": 1024 }, - "v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.vk": { - "cid": "QmRDcUxfpPY9a1vR3T4vRgxHtWHyy9m3xuMQtj8P749r4e", - "digest": "3632776cd23e376694c625390b9a73ea", + "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": { + "cid": "QmTGQGThNFEjaMFgP32YLubACrtpkRoeVEfhDaWi5g6w8u", + "digest": "021b3e81e2980a50fd1ac07424d29a8d", "sector_size": 1024 } -} +} \ No newline at end of file diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 29891fde6..f956870ed 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -8,13 +8,18 @@ import ( "github.com/ipfs/go-datastore" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sector" ) func TestSealAndVerify(t *testing.T) { - t.Skip("this is slow") + //t.Skip("this is slow") + if err := build.GetParams(true); err != nil { + t.Fatal(err) + } + dir, err := ioutil.TempDir("", "sbtest") if err != nil { t.Fatal(err) @@ -46,7 +51,7 @@ func TestSealAndVerify(t *testing.T) { store.Service() ssinfo := <-store.Incoming() - ok, err := sectorbuilder.VerifySeal(1024, ssinfo.CommR[:], ssinfo.CommD[:], ssinfo.CommRStar[:], addr, ssinfo.SectorID, ssinfo.Proof) + ok, err := sectorbuilder.VerifySeal(1024, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) if err != nil { t.Fatal(err) } From 874be799581e8d24e29b4cabe72b70fc4f3669c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 27 Oct 2019 09:56:53 +0100 Subject: [PATCH 041/230] very basic sector seal scheduling --- api/api.go | 3 --- api/struct.go | 6 ------ chain/deals/provider_states.go | 4 ++++ cmd/lotus-storage-miner/sectors.go | 16 --------------- lib/sectorbuilder/sectorbuilder.go | 10 +++++---- lib/sectorbuilder/sectorbuilder_test.go | 3 ++- lotuspond/front/src/StorageNode.js | 4 +++- node/builder.go | 1 + node/impl/storminer.go | 11 +++++----- node/modules/storageminer.go | 27 +++++++++++++++++++++++++ storage/miner.go | 1 + storage/sector/store.go | 25 ++++++++++++++++++++--- 12 files changed, 71 insertions(+), 40 deletions(-) diff --git a/api/api.go b/api/api.go index 77edad959..3d68fdc49 100644 --- a/api/api.go +++ b/api/api.go @@ -167,9 +167,6 @@ type StorageMiner interface { // List all staged sectors SectorsList(context.Context) ([]uint64, error) - // Seal all staged sectors - SectorsStagedSeal(context.Context) error - SectorsRefs(context.Context) (map[string][]SealedRef, error) } diff --git a/api/struct.go b/api/struct.go index ce4bba8ef..7d738dad7 100644 --- a/api/struct.go +++ b/api/struct.go @@ -131,7 +131,6 @@ type StorageMinerStruct struct { SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` SectorsList func(context.Context) ([]uint64, error) `perm:"read"` - SectorsStagedSeal func(context.Context) error `perm:"write"` SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` } @@ -476,11 +475,6 @@ func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]uint64, error) return c.Internal.SectorsList(ctx) } -// Seal all staged sectors -func (c *StorageMinerStruct) SectorsStagedSeal(ctx context.Context) error { - return c.Internal.SectorsStagedSeal(ctx) -} - func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) { return c.Internal.SectorsRefs(ctx) } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index e26775439..c2f49d4f0 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -273,6 +273,10 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal log.Warnf("Sending deal response failed: %s", err) } + if err := p.secst.SealSector(ctx, deal.SectorID); err != nil { + return nil, xerrors.Errorf("sealing sector failed: %w", err) + } + _, err = p.waitSealed(ctx, deal) if err != nil { return nil, err diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index dd7e0ac47..149e8195d 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -36,7 +36,6 @@ var sectorsCmd = &cli.Command{ Subcommands: []*cli.Command{ sectorsStatusCmd, sectorsStagedListCmd, - sectorsStagedSealCmd, sectorsRefsCmd, }, } @@ -102,21 +101,6 @@ var sectorsStagedListCmd = &cli.Command{ }, } -var sectorsStagedSealCmd = &cli.Command{ - Name: "seal-staged", // TODO: nest this under a 'staged' subcommand? idk - Usage: "Seal staged sectors", - Action: func(cctx *cli.Context) error { - nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - ctx := lcli.ReqContext(cctx) - - return nodeApi.SectorsStagedSeal(ctx) - }, -} - var sectorsRefsCmd = &cli.Command{ Name: "refs", Usage: "List References to sectors", diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index a9790e9ba..b3ad7d4be 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -25,6 +25,10 @@ type SortedSectorInfo = sectorbuilder.SortedSectorInfo type SectorInfo = sectorbuilder.SectorInfo +type SealTicket = sectorbuilder.SealTicket + +type SealedSectorMetadata = sectorbuilder.SealedSectorMetadata + const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { @@ -87,10 +91,8 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, err return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } -func (sb *SectorBuilder) SealAllStagedSectors() error { - panic("dont call this") - _, err := sectorbuilder.SealAllStagedSectors(sb.handle, sectorbuilder.SealTicket{}) - return err +func (sb *SectorBuilder) SealSector(sectorID uint64, ticket SealTicket) (SealedSectorMetadata, error) { + return sectorbuilder.SealSector(sb.handle, sectorID, ticket) } func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index f956870ed..a2202d585 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -47,7 +47,8 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - store := sector.NewStore(sb, datastore.NewMapDatastore()) + // TODO: Consider fixing + store := sector.NewStore(sb, datastore.NewMapDatastore(), nil) store.Service() ssinfo := <-store.Incoming() diff --git a/lotuspond/front/src/StorageNode.js b/lotuspond/front/src/StorageNode.js index ef52452f9..741932db9 100644 --- a/lotuspond/front/src/StorageNode.js +++ b/lotuspond/front/src/StorageNode.js @@ -13,6 +13,8 @@ let sealCodes = [ "Failed", "Sealing", "Sealed", + "Paused", + "ReadyForSealing", ] class StorageNode extends React.Component { @@ -122,7 +124,7 @@ class StorageNode extends React.Component {
{this.state.statusCounts.map((c, i) => {sealCodes[i]}: {c} | )}
{this.state.staged ? this.state.staged.map((s, i) => ( -
{s.SectorID} {sealCodes[s.State]}
+
{s.SectorID} {sealCodes[s.State] || `unk ${s.State}`}
)) :
}
diff --git a/node/builder.go b/node/builder.go index d5739c472..c316139ee 100644 --- a/node/builder.go +++ b/node/builder.go @@ -235,6 +235,7 @@ func Online() Option { Override(new(*sector.Store), sector.NewStore), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(*commitment.Tracker), commitment.NewTracker), + Override(new(sector.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 5631e8e09..e70480651 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -40,11 +40,15 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) // TODO: create a deal name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), 0) + sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) if err != nil { return 0, err } + if err := sm.Sectors.SealSector(ctx, sectorId); err != nil { + return sectorId, err + } + return sectorId, err } @@ -57,11 +61,6 @@ func (sm *StorageMinerAPI) SectorsList(context.Context) ([]uint64, error) { return sm.SectorBuilder.GetAllStagedSectors() } -// Seal all staged sectors -func (sm *StorageMinerAPI) SectorsStagedSeal(context.Context) error { - return sm.SectorBuilder.SealAllStagedSectors() -} - func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { // json can't handle cids as map keys out := map[string][]api.SealedRef{} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4b7e13b94..74b7529f6 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -14,8 +14,10 @@ import ( "github.com/libp2p/go-libp2p-core/routing" "github.com/mitchellh/go-homedir" "go.uber.org/fx" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/lib/sectorbuilder" @@ -161,3 +163,28 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro }) return nil } + +func SealTicketGen(api api.FullNode) sector.TicketFn { + return func(ctx context.Context) (*sectorbuilder.SealTicket, error) { + ts, err := api.ChainHead(ctx) + if err != nil { + return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err) + } + + r, err := api.ChainGetRandomness(ctx, ts, nil, build.PoSTChallangeTime) + if err != nil { + return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err) + } + + var tkt [sectorbuilder.CommLen]byte + if n := copy(tkt[:], r); n != sectorbuilder.CommLen { + return nil, xerrors.Errorf("unexpected randomness len: %d (expected %d)", n, sectorbuilder.CommLen) + } + + + return §orbuilder.SealTicket{ + BlockHeight: ts.Height(), + TicketBytes: tkt, + }, nil + } +} diff --git a/storage/miner.go b/storage/miner.go index 599634ddb..0ab1ce009 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -144,6 +144,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal CommD: sinfo.CommD[:], CommR: sinfo.CommR[:], Proof: sinfo.Proof, + Epoch: sinfo.Ticket.BlockHeight, DealIDs: deals, SectorNumber: sinfo.SectorID, diff --git a/storage/sector/store.go b/storage/sector/store.go index 65ccd65ce..0a39fa0a0 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -32,11 +32,14 @@ type dealMapping struct { Committed bool } +type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) + // TODO: eventually handle sector storage here instead of in rust-sectorbuilder type Store struct { waitingLk sync.Mutex sb *sectorbuilder.SectorBuilder + tktFn TicketFn dealsLk sync.Mutex deals datastore.Datastore @@ -48,9 +51,10 @@ type Store struct { closeCh chan struct{} } -func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS) *Store { +func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store { return &Store{ sb: sb, + tktFn:tktFn, deals: namespace.Wrap(ds, sectorDealsPrefix), waiting: map[uint64]chan struct{}{}, closeCh: make(chan struct{}), @@ -121,7 +125,7 @@ func (s *Store) service() { } } -func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealID uint64) (sectorID uint64, err error) { +func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { sectorID, err = s.sb.AddPiece(ref, size, r) if err != nil { return 0, err @@ -150,7 +154,7 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealID uint64) (s } fallthrough case datastore.ErrNotFound: - deals.DealIDs = append(deals.DealIDs, dealID) + deals.DealIDs = append(deals.DealIDs, dealIDs...) d, err := cbor.DumpObject(&deals) if err != nil { return 0, err @@ -200,6 +204,21 @@ func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { } } +func (s *Store) SealSector(ctx context.Context, sectorID uint64) error { + tkt, err := s.tktFn(ctx) + if err != nil { + return err + } + + // TODO: That's not async, is it? + // - If not then we probably can drop this wait-for-seal hack below + _, err = s.sb.SealSector(sectorID, *tkt) + if err != nil { + return err + } + return nil +} + func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { s.waitingLk.Lock() var at = -1 From 117ec636c5aab7debdee137203a7078975f17eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 27 Oct 2019 10:18:27 +0100 Subject: [PATCH 042/230] paramfetch: check checksums in parallel --- api/struct.go | 4 +- build/paramfetch.go | 80 ++++++++++++++++++++++++--------- chain/gen/utils.go | 2 +- chain/types/blockheader_test.go | 2 +- go.mod | 1 + node/modules/storageminer.go | 1 - storage/sector/store.go | 4 +- 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/api/struct.go b/api/struct.go index 7d738dad7..6482a3e5b 100644 --- a/api/struct.go +++ b/api/struct.go @@ -129,8 +129,8 @@ type StorageMinerStruct struct { StoreGarbageData func(context.Context) (uint64, error) `perm:"write"` - SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` - SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` + SectorsList func(context.Context) ([]uint64, error) `perm:"read"` SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` } diff --git a/build/paramfetch.go b/build/paramfetch.go index bf3e5b719..abfeca08b 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -8,10 +8,13 @@ import ( "os" "path/filepath" "strings" + "sync" rice "github.com/GeertJohan/go.rice" logging "github.com/ipfs/go-log" "github.com/minio/blake2b-simd" + "go.uber.org/multierr" + "golang.org/x/xerrors" pb "gopkg.in/cheggaaa/pb.v1" ) @@ -27,6 +30,13 @@ type paramFile struct { SectorSize uint64 `json:"sector_size"` } +type fetch struct { + wg sync.WaitGroup + fetchLk sync.Mutex + + errs []error +} + func GetParams(storage bool) error { if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) { return err @@ -39,6 +49,8 @@ func GetParams(storage bool) error { return err } + ft := &fetch{} + for name, info := range params { if !SupportedSectorSize(info.SectorSize) { continue @@ -47,35 +59,61 @@ func GetParams(storage bool) error { continue } - if err := maybeFetch(name, info); err != nil { - return err - } + ft.maybeFetchAsync(name, info) } - return nil + return ft.wait() } -func maybeFetch(name string, info paramFile) error { - path := filepath.Join(paramdir, name) +func (ft *fetch) maybeFetchAsync(name string, info paramFile) { + ft.wg.Add(1) + + go func() { + defer ft.wg.Done() + + path := filepath.Join(paramdir, name) + + err := ft.checkFile(path, info) + if !os.IsNotExist(err) && err != nil { + log.Warn(err) + } + if err == nil { + return + } + + ft.fetchLk.Lock() + defer ft.fetchLk.Unlock() + + if err := doFetch(path, info); err != nil { + ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s: %w", path, err)) + } + }() +} + +func (ft *fetch) checkFile(path string, info paramFile) error { f, err := os.Open(path) - if err == nil { - defer f.Close() + if err != nil { + return err + } + defer f.Close() - h := blake2b.New512() - if _, err := io.Copy(h, f); err != nil { - return err - } - - sum := h.Sum(nil) - strSum := hex.EncodeToString(sum[:16]) - if strSum == info.Digest { - return nil - } - - log.Warnf("Checksum mismatch in param file %s, %s != %s", name, strSum, info.Digest) + h := blake2b.New512() + if _, err := io.Copy(h, f); err != nil { + return err } - return doFetch(path, info) + sum := h.Sum(nil) + strSum := hex.EncodeToString(sum[:16]) + if strSum == info.Digest { + return nil + } + + return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest) +} + +func (ft *fetch) wait() error { + ft.wg.Wait() + return multierr.Combine(ft.errs...) } func doFetch(out string, info paramFile) error { diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 436160da0..303878c94 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -372,7 +372,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B log.Infof("Empty Genesis root: %s", emptyroot) genesisticket := &types.Ticket{ - VRFProof: []byte("vrf proof"), + VRFProof: []byte("vrf proof0000000vrf proof0000000"), } b := &types.BlockHeader{ diff --git a/chain/types/blockheader_test.go b/chain/types/blockheader_test.go index d32e8bdce..09ca061b9 100644 --- a/chain/types/blockheader_test.go +++ b/chain/types/blockheader_test.go @@ -27,7 +27,7 @@ func testBlockHeader(t testing.TB) *BlockHeader { ElectionProof: []byte("cats won the election"), Tickets: []*Ticket{ &Ticket{ - VRFProof: []byte("vrf proof"), + VRFProof: []byte("vrf proof0000000vrf proof0000000"), }, }, Parents: []cid.Cid{c, c}, diff --git a/go.mod b/go.mod index d72aaa4cd..343315e1c 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( go.uber.org/dig v1.7.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect + go.uber.org/multierr v1.1.0 go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 74b7529f6..bda65e9db 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -181,7 +181,6 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { return nil, xerrors.Errorf("unexpected randomness len: %d (expected %d)", n, sectorbuilder.CommLen) } - return §orbuilder.SealTicket{ BlockHeight: ts.Height(), TicketBytes: tkt, diff --git a/storage/sector/store.go b/storage/sector/store.go index 0a39fa0a0..5a4b6a9a3 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -38,7 +38,7 @@ type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) type Store struct { waitingLk sync.Mutex - sb *sectorbuilder.SectorBuilder + sb *sectorbuilder.SectorBuilder tktFn TicketFn dealsLk sync.Mutex @@ -54,7 +54,7 @@ type Store struct { func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store { return &Store{ sb: sb, - tktFn:tktFn, + tktFn: tktFn, deals: namespace.Wrap(ds, sectorDealsPrefix), waiting: map[uint64]chan struct{}{}, closeCh: make(chan struct{}), From c1076a0c7d3d95560537b753130722c52632df9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 27 Oct 2019 10:24:20 +0100 Subject: [PATCH 043/230] set correct block height in SealTicket --- node/modules/storageminer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index bda65e9db..ff51ca7b8 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -171,7 +171,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err) } - r, err := api.ChainGetRandomness(ctx, ts, nil, build.PoSTChallangeTime) + r, err := api.ChainGetRandomness(ctx, ts, nil, build.RandomnessLookback) if err != nil { return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err) } @@ -182,7 +182,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { } return §orbuilder.SealTicket{ - BlockHeight: ts.Height(), + BlockHeight: ts.Height() - build.RandomnessLookback, TicketBytes: tkt, }, nil } From 53cbe3446dfd7d5d00754958aaf8b4bd0d3e2cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 27 Oct 2019 11:27:21 +0100 Subject: [PATCH 044/230] pond: Miner state viewer --- api/api.go | 2 +- api/struct.go | 6 ++-- cli/state.go | 2 +- lotuspond/front/src/Block.js | 2 +- lotuspond/front/src/State.js | 69 ++++++++++++++++++++++++++++++++++-- node/impl/full/state.go | 4 +-- 6 files changed, 74 insertions(+), 11 deletions(-) diff --git a/api/api.go b/api/api.go index 3d68fdc49..94f69485c 100644 --- a/api/api.go +++ b/api/api.go @@ -123,7 +123,7 @@ type FullNode interface { StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) - StateMinerSectors(context.Context, address.Address) ([]*SectorInfo, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) StateMinerPower(context.Context, address.Address, *types.TipSet) (MinerPower, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) diff --git a/api/struct.go b/api/struct.go index 6482a3e5b..3fbbf619c 100644 --- a/api/struct.go +++ b/api/struct.go @@ -86,7 +86,7 @@ type FullNodeStruct struct { ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address) ([]*SectorInfo, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` @@ -338,8 +338,8 @@ func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMs return c.Internal.SyncSubmitBlock(ctx, blk) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address) ([]*SectorInfo, error) { - return c.Internal.StateMinerSectors(ctx, addr) +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { + return c.Internal.StateMinerSectors(ctx, addr, ts) } func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { diff --git a/cli/state.go b/cli/state.go index b40611c54..6d831b249 100644 --- a/cli/state.go +++ b/cli/state.go @@ -78,7 +78,7 @@ var stateSectorsCmd = &cli.Command{ return err } - sectors, err := api.StateMinerSectors(ctx, maddr) + sectors, err := api.StateMinerSectors(ctx, maddr, nil) if err != nil { return err } diff --git a/lotuspond/front/src/Block.js b/lotuspond/front/src/Block.js index c7f7ae682..44104d483 100644 --- a/lotuspond/front/src/Block.js +++ b/lotuspond/front/src/Block.js @@ -78,7 +78,7 @@ class Block extends React.Component { ) } - return ( + return ( {content} ) } diff --git a/lotuspond/front/src/State.js b/lotuspond/front/src/State.js index e1fb42609..6fb1a70aa 100644 --- a/lotuspond/front/src/State.js +++ b/lotuspond/front/src/State.js @@ -4,12 +4,14 @@ import CID from "cids"; import * as multihash from "multihashes"; import code from "./chain/code"; import Address from "./Address"; +import Fil from "./Fil"; class State extends React.Component { byCode = { [code.init]: InitState, [code.power]: PowerState, [code.market]: MarketState, + [code.miner]: MinerState, } constructor(props) { @@ -33,17 +35,17 @@ class State extends React.Component { let state if(this.byCode[this.state.code]) { const Stelem = this.byCode[this.state.code] - state = + state = } else { state =
{Object.keys(this.state.State).map(k =>
{k}: {JSON.stringify(this.state.State[k])}
)}
} const content =
-
Balance: {this.state.Balance}
+
Balance: {this.state.Balance}
---
{state}
- return + return {content} } @@ -133,4 +135,65 @@ class MarketState extends React.Component { } } +class MinerState extends React.Component { + constructor(props) { + super(props) + this.state = {state: {}, sectorSize: -1, worker: "", networkPower: 0, sectors: {}} + } + + async componentDidMount() { + const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props + + const state = await this.props.client.call('Filecoin.StateReadState', [this.props.actor, tipset]) + const sectorSize = await this.props.client.call("Filecoin.StateMinerSectorSize", [this.props.addr, tipset]) + const worker = await this.props.client.call("Filecoin.StateMinerWorker", [this.props.addr, tipset]) + + const tpow = await this.props.client.call("Filecoin.StateMinerPower", [this.props.addr, tipset]) + const networkPower = tpow.TotalPower + + let sectors = {} + + const sset = await this.props.client.call("Filecoin.StateMinerSectors", [this.props.addr, tipset]) || [] + const pset = await this.props.client.call("Filecoin.StateMinerProvingSet", [this.props.addr, tipset]) || [] + + sset.forEach(s => sectors[s.SectorID] = {...s, sectorSet: true}) + pset.forEach(s => sectors[s.SectorID] = {...(sectors[s.SectorID] || s), provingSet: true}) + + this.setState({state, sectorSize, worker, networkPower, sectors}) + } + + render() { + if (!this.state.worker) { + return (...) + } + + let state = this.state.state.State + + return
+
Worker:
+
Sector Size: {this.state.sectorSize/1024} KiB
+
Power: {state.Power} ({state.Power/this.state.networkPower*100}%)
+
Proving Period End: {state.ProvingPeriodEnd}
+
+
----
+
Sectors:
+ + + + + + {Object.keys(this.state.sectors).map(sid => + + + + + + )} + +
IDCommDCommRSectorSetProving
{sid}{this.state.sectors[sid].CommD}{this.state.sectors[sid].CommR}{this.state.sectors[sid].sectorSet ? 'X' : ' '}{this.state.sectors[sid].provingSet ? 'X' : ' '}
+
+
+ } +} + export default State \ No newline at end of file diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 45f2aa94e..68a134246 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -37,8 +37,8 @@ type StateAPI struct { Chain *store.ChainStore } -func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address) ([]*api.SectorInfo, error) { - return stmgr.GetMinerSectorSet(ctx, a.StateManager, nil, addr) +func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { + return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr) } func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { From 02b67ea6b201dbd47c1eed700e1a5f24a058fa92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 27 Oct 2019 12:58:15 +0100 Subject: [PATCH 045/230] Fix TestSealAndVerify --- lib/sectorbuilder/sectorbuilder.go | 9 +---- lib/sectorbuilder/sectorbuilder_test.go | 47 ++++++++++++++++++------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index b3ad7d4be..c9cad6ae5 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -1,7 +1,6 @@ package sectorbuilder import ( - "encoding/binary" "io" "os" "sort" @@ -46,7 +45,7 @@ type SectorBuilderConfig struct { func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 1, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) if err != nil { return nil, err } @@ -62,12 +61,6 @@ func addressToProverID(a address.Address) [32]byte { return proverId } -func sectorIDtoBytes(sid uint64) [31]byte { - var out [31]byte - binary.LittleEndian.PutUint64(out[:], sid) - return out -} - func (sb *SectorBuilder) Destroy() { sectorbuilder.DestroySectorBuilder(sb.handle) } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index a2202d585..b40a0b9cd 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -1,9 +1,11 @@ package sectorbuilder_test import ( + "context" "io" "io/ioutil" "math/rand" + "path/filepath" "testing" "github.com/ipfs/go-datastore" @@ -14,8 +16,12 @@ import ( "github.com/filecoin-project/lotus/storage/sector" ) +const sectorSize = 1024 + func TestSealAndVerify(t *testing.T) { - //t.Skip("this is slow") + t.Skip("this is slow") + build.SectorSizes = []uint64{sectorSize} + if err := build.GetParams(true); err != nil { t.Fatal(err) } @@ -25,34 +31,51 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - addr, err := address.NewFromString("t1tct3nfaw2q543xtybxcyw4deyxmfwkjk43u4t5y") + addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") if err != nil { t.Fatal(err) } + metadata := filepath.Join(dir, "meta") + sealed := filepath.Join(dir, "sealed") + staging := filepath.Join(dir, "staging") + sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ - SectorSize: 1024, - SealedDir: dir, - StagedDir: dir, - MetadataDir: dir, + SectorSize: sectorSize, + SealedDir: sealed, + StagedDir: staging, + MetadataDir: metadata, Miner: addr, }) if err != nil { t.Fatal(err) } - r := io.LimitReader(rand.New(rand.NewSource(42)), 1016) + // TODO: Consider fixing + store := sector.NewStore(sb, datastore.NewMapDatastore(), func(ctx context.Context) (*sectorbuilder.SealTicket, error) { + return §orbuilder.SealTicket{ + BlockHeight: 5, + TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, + }, nil + }) - if _, err := sb.AddPiece("foo", 1016, r); err != nil { + store.Service() + + dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) + + r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) + sid, err := store.AddPiece("foo", dlen, r) + if err != nil { + t.Fatal(err) + } + + if err := store.SealSector(context.TODO(), sid); err != nil { t.Fatal(err) } - // TODO: Consider fixing - store := sector.NewStore(sb, datastore.NewMapDatastore(), nil) - store.Service() ssinfo := <-store.Incoming() - ok, err := sectorbuilder.VerifySeal(1024, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) + ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) if err != nil { t.Fatal(err) } From 0822bdadacf21d73c634793b03e5ddb8ca778d94 Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Mon, 28 Oct 2019 19:29:16 +1000 Subject: [PATCH 046/230] add statemanager.CallRaw span This allows for the chain.Sync trace to be connected to the entire chain sync operation, when viewing in Jaeger. --- chain/stmgr/call.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index fffc923c2..731633d26 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/ipfs/go-cid" + "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/actors" @@ -14,6 +15,9 @@ import ( ) func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight uint64) (*types.MessageReceipt, error) { + ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw") + defer span.End() + vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore()) if err != nil { return nil, xerrors.Errorf("failed to set up vm: %w", err) @@ -29,6 +33,14 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate msg.Value = types.NewInt(0) } + if span.IsRecordingEvents() { + span.AddAttributes( + trace.Int64Attribute("gas_limit", int64(msg.GasLimit.Uint64())), + trace.Int64Attribute("gas_price", int64(msg.GasPrice.Uint64())), + trace.StringAttribute("value", msg.Value.String()), + ) + } + fromActor, err := vmi.StateTree().GetActor(msg.From) if err != nil { return nil, xerrors.Errorf("call raw get actor: %s", err) From 237f8e8018ad864ada961d58bcbb9c958332093f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Oct 2019 09:15:08 +0100 Subject: [PATCH 047/230] Set proving params matching the spec more closely --- build/params.go | 29 ++++++++++++++++------------- chain/actors/actor_miner.go | 4 ++-- chain/gen/gen.go | 2 +- chain/sync.go | 2 +- storage/post.go | 12 ++++++------ 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/build/params.go b/build/params.go index 9185a5b0c..480b90230 100644 --- a/build/params.go +++ b/build/params.go @@ -33,14 +33,6 @@ func SupportedSectorSize(ssize uint64) bool { // Blocks const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours -// Blocks -const DealVoucherSkewLimit = 10 - -// Blocks -const MinDealVoucherIncrement = ProvingPeriodDuration - -const MaxVouchersPerDeal = 768 // roughly one voucher per 10h over a year - // ///// // Consensus / Network @@ -57,16 +49,27 @@ const ForkLengthThreshold = 100 const BlocksPerEpoch = 1 // ///// -// Proofs / Mining - -// Blocks -const RandomnessLookback = 20 +// Proofs // Blocks const ProvingPeriodDuration = 40 +// PoStChallangeTime sets the window in which post computation should happen // Blocks -const PoSTChallangeTime = 35 +const PoStChallangeTime = ProvingPeriodDuration - 5 + +// PoStRandomnessLookback is additional randomness lookback for PoSt computation +// To compute randomness epoch in a given proving period: +// RandH = PPE - PoStChallangeTime - PoStRandomnessLookback +// +// Blocks +const PoStRandomnessLookback = 1 + +// ///// +// Mining + +// Blocks +const EcRandomnessLookback = 300 const PowerCollateralProportion = 5 const PerCapitaCollateralProportion = 1 diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index dc5f8c530..baa35d0d5 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -360,7 +360,7 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext, var seed [sectorbuilder.CommLen]byte { - randHeight := currentProvingPeriodEnd - build.PoSTChallangeTime + randHeight := currentProvingPeriodEnd - build.PoStChallangeTime - build.PoStRandomnessLookback if vmctx.BlockHeight() <= randHeight { // TODO: spec, retcode return nil, aerrors.Newf(1, "submit PoSt called outside submission window (%d < %d)", vmctx.BlockHeight(), randHeight) @@ -658,7 +658,7 @@ func (sma StorageMinerActor) AddFaults(act *types.Actor, vmctx types.VMContext, return nil, aerr } - challengeHeight := self.ProvingPeriodEnd - build.PoSTChallangeTime + challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime if vmctx.BlockHeight() < challengeHeight { // TODO: optimized bitfield methods diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 6032dd61c..91d7947c4 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -399,7 +399,7 @@ func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*ty } func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, miner address.Address, a MiningCheckAPI) (bool, types.ElectionProof, error) { - r, err := a.ChainGetRandomness(ctx, ts, ticks, build.RandomnessLookback) + r, err := a.ChainGetRandomness(ctx, ts, ticks, build.EcRandomnessLookback) if err != nil { return false, nil, xerrors.Errorf("chain get randomness: %w", err) } diff --git a/chain/sync.go b/chain/sync.go index 40af03964..dac2a79e5 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -495,7 +495,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("validating block tickets failed: %w", err) } - rand, err := syncer.sm.ChainStore().GetRandomness(ctx, baseTs.Cids(), h.Tickets, build.RandomnessLookback) + rand, err := syncer.sm.ChainStore().GetRandomness(ctx, baseTs.Cids(), h.Tickets, build.EcRandomnessLookback) if err != nil { return xerrors.Errorf("failed to get randomness for verifying election proof: %w", err) } diff --git a/storage/post.go b/storage/post.go index d34069894..ee024acbb 100644 --- a/storage/post.go +++ b/storage/post.go @@ -44,12 +44,12 @@ func (m *Miner) beginPosting(ctx context.Context) { m.schedLk.Unlock() - log.Infof("Scheduling post at height %d", ppe-build.PoSTChallangeTime) + log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime) err = m.events.ChainAt(m.computePost(m.schedPost), func(ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil - }, PoStConfidence, ppe-build.PoSTChallangeTime) + }, PoStConfidence, ppe-build.PoStChallangeTime) if err != nil { // TODO: This is BAD, figure something out log.Errorf("scheduling PoSt failed: %s", err) @@ -82,13 +82,13 @@ func (m *Miner) scheduleNextPost(ppe uint64) { m.schedPost = ppe m.schedLk.Unlock() - log.Infow("scheduling PoSt", "post-height", ppe-build.PoSTChallangeTime, + log.Infow("scheduling PoSt", "post-height", ppe-build.PoStChallangeTime, "height", ts.Height(), "ppe", ppe, "proving-period", provingPeriod) err = m.events.ChainAt(m.computePost(ppe), func(ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil - }, PoStConfidence, ppe-build.PoSTChallangeTime) + }, PoStConfidence, ppe-build.PoStChallangeTime) if err != nil { // TODO: This is BAD, figure something out log.Errorf("scheduling PoSt failed: %+v", err) @@ -113,13 +113,13 @@ func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) erro return xerrors.Errorf("failed to get proving set for miner: %w", err) } - r, err := m.api.ChainGetRandomness(ctx, ts, nil, int(int64(ts.Height())-int64(ppe)+int64(build.PoSTChallangeTime))) // TODO: review: check math + r, err := m.api.ChainGetRandomness(ctx, ts, nil, int(int64(ts.Height())-int64(ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math if err != nil { return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", ts.Height(), ppe, err) } log.Infow("running PoSt", "delayed-by", - int64(ts.Height())-(int64(ppe)-int64(build.PoSTChallangeTime)), + int64(ts.Height())-(int64(ppe)-int64(build.PoStChallangeTime)), "chain-random", r, "ppe", ppe, "height", ts.Height()) tsStart := time.Now() From ac98c8f45145ef8de7ec23625ddf84ef9b80b3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Oct 2019 18:14:24 +0100 Subject: [PATCH 048/230] Use spec complaiant SealRandomnessLookback --- build/params.go | 6 ++++++ node/modules/storageminer.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build/params.go b/build/params.go index 480b90230..00808dbe4 100644 --- a/build/params.go +++ b/build/params.go @@ -48,6 +48,9 @@ const ForkLengthThreshold = 100 // Blocks (e) const BlocksPerEpoch = 1 +// Blocks +const Finality = 500 + // ///// // Proofs @@ -65,6 +68,9 @@ const PoStChallangeTime = ProvingPeriodDuration - 5 // Blocks const PoStRandomnessLookback = 1 +// Blocks +const SealRandomnessLookback = Finality + // ///// // Mining diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index ff51ca7b8..4f9df1c42 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -171,7 +171,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err) } - r, err := api.ChainGetRandomness(ctx, ts, nil, build.RandomnessLookback) + r, err := api.ChainGetRandomness(ctx, ts, nil, build.SealRandomnessLookback) if err != nil { return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err) } @@ -182,7 +182,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { } return §orbuilder.SealTicket{ - BlockHeight: ts.Height() - build.RandomnessLookback, + BlockHeight: ts.Height() - build.SealRandomnessLookback, TicketBytes: tkt, }, nil } From 15742f235952e92594aa0fb09e07f175d33a9272 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 23 Oct 2019 23:05:53 -0700 Subject: [PATCH 049/230] Remove height tag from stats tool Having a height tags resulted in performance issues due to an increase in series cardinality. --- tools/stats/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/stats/main.go b/tools/stats/main.go index f77ba941a..c8d60de0c 100644 --- a/tools/stats/main.go +++ b/tools/stats/main.go @@ -3,7 +3,6 @@ package main import ( "context" "flag" - "fmt" "log" "os" "time" @@ -94,7 +93,6 @@ func main() { // Instead of having to pass around a bunch of generic stuff we want for each point // we will just add them at the end. - tsHeight := fmt.Sprintf("%d", tipset.Height()) tsTimestamp := time.Unix(int64(tipset.MinTimestamp()), int64(0)) nb, err := InfluxNewBatch() @@ -103,7 +101,6 @@ func main() { } for _, pt := range pl.Points() { - pt.AddTag("height", tsHeight) pt.SetTime(tsTimestamp) nb.AddPoint(NewPointFrom(pt)) From 6d766ab241b204cf4868fad0b5021ec3f1671ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Oct 2019 19:22:40 +0100 Subject: [PATCH 050/230] Set e=3 --- build/params.go | 2 +- chain/gen/gen.go | 2 +- chain/types/blockheader.go | 5 ++++- chain/vm/vm.go | 5 +++++ lotuspond/front/src/BlockLink.js | 2 +- lotuspond/front/src/ChainExplorer.js | 2 +- lotuspond/front/src/Consensus.js | 2 +- 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/build/params.go b/build/params.go index 00808dbe4..0e19e4972 100644 --- a/build/params.go +++ b/build/params.go @@ -46,7 +46,7 @@ const AllowableClockDrift = BlockDelay * 2 const ForkLengthThreshold = 100 // Blocks (e) -const BlocksPerEpoch = 1 +const BlocksPerEpoch = 3 // Blocks const Finality = 500 diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 91d7947c4..b38f2c0df 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -419,7 +419,7 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, return false, nil, xerrors.Errorf("failed to check power: %w", err) } - return types.PowerCmp(vrfout, types.BigMul(pow.MinerPower, types.NewInt(build.BlocksPerEpoch)), pow.TotalPower), vrfout, nil + return types.PowerCmp(vrfout, pow.MinerPower, pow.TotalPower), vrfout, nil } type SignFunc func(context.Context, address.Address, []byte) (*types.Signature, error) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index a82bf37a2..6109df28f 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -12,6 +12,7 @@ import ( "go.opencensus.io/trace" xerrors "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" ) @@ -159,6 +160,8 @@ func CidArrsEqual(a, b []cid.Cid) bool { return true } +var blocksPerEpoch = NewInt(build.BlocksPerEpoch) + func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { /* @@ -171,7 +174,7 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { // 2^256 rden := BigInt{big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil)} - top := BigMul(rden, mpow) + top := BigMul(BigMul(rden, mpow), blocksPerEpoch) out := BigDiv(top, totpow) hp := BigFromBytes(h[:]) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 1cfcab905..8b702c48b 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -488,6 +488,9 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet, return nil, xerrors.Errorf("getting block miner actor (%s) failed: %w", vm.blockMiner, err) } + + // TODO: support multiple blocks in a tipset + // TODO: actually wire this up (miner is undef for now) gasReward := types.BigMul(msg.GasPrice, gasUsed) if err := Transfer(gasHolder, miner, gasReward); err != nil { return nil, xerrors.Errorf("failed to give miner gas reward: %w", err) @@ -630,6 +633,7 @@ func depositFunds(act *types.Actor, amt types.BigInt) { } var miningRewardTotal = types.FromFil(build.MiningRewardTotal) +var blocksPerEpoch = types.NewInt(build.BlocksPerEpoch) // MiningReward returns correct mining reward // coffer is amount of FIL in NetworkAddress @@ -637,5 +641,6 @@ func MiningReward(remainingReward types.BigInt) types.BigInt { ci := big.NewInt(0).Set(remainingReward.Int) res := ci.Mul(ci, build.InitialReward) res = res.Div(res, miningRewardTotal.Int) + res = res.Div(res, blocksPerEpoch.Int) return types.BigInt{res} } diff --git a/lotuspond/front/src/BlockLink.js b/lotuspond/front/src/BlockLink.js index 1faac124c..bfdbd5ea9 100644 --- a/lotuspond/front/src/BlockLink.js +++ b/lotuspond/front/src/BlockLink.js @@ -12,7 +12,7 @@ export class BlockLinks extends React.Component { block = this.props.blocks[k] } - return + return }) } } diff --git a/lotuspond/front/src/ChainExplorer.js b/lotuspond/front/src/ChainExplorer.js index 1f8f619c0..8957cb3d0 100644 --- a/lotuspond/front/src/ChainExplorer.js +++ b/lotuspond/front/src/ChainExplorer.js @@ -169,7 +169,7 @@ class ChainExplorer extends React.Component { return
@{h} {info}
})}
- return ( + return ( {content} ) } diff --git a/lotuspond/front/src/Consensus.js b/lotuspond/front/src/Consensus.js index 86b1362a4..14b4dd6e2 100644 --- a/lotuspond/front/src/Consensus.js +++ b/lotuspond/front/src/Consensus.js @@ -5,7 +5,7 @@ import Window from "./Window"; function styleForHDiff(max, act) { switch (max - act) { case 0: - return {background: '#00aa00'} + return {background: '#004400'} case 1: return {background: '#aaaa00'} default: From 6f73e8bf2f19c5dc18f3f54655d2abb19d94e36a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Oct 2019 19:25:24 +0100 Subject: [PATCH 051/230] 10s block time --- build/params.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/params.go b/build/params.go index 0e19e4972..e810ff45f 100644 --- a/build/params.go +++ b/build/params.go @@ -37,7 +37,7 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // Consensus / Network // Seconds -const BlockDelay = 3 +const BlockDelay = 10 // Seconds const AllowableClockDrift = BlockDelay * 2 @@ -55,11 +55,11 @@ const Finality = 500 // Proofs // Blocks -const ProvingPeriodDuration = 40 +const ProvingPeriodDuration = 60 // PoStChallangeTime sets the window in which post computation should happen // Blocks -const PoStChallangeTime = ProvingPeriodDuration - 5 +const PoStChallangeTime = ProvingPeriodDuration - 6 // PoStRandomnessLookback is additional randomness lookback for PoSt computation // To compute randomness epoch in a given proving period: From 563c0e494e7181b0beacb406a91b8541e5cb9cbb Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 28 Oct 2019 20:01:10 +0100 Subject: [PATCH 052/230] Use div-free EC power selection function License: MIT Signed-off-by: Jakub Sztandera --- chain/types/blockheader.go | 21 +++++++++++++-------- go.mod | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 6109df28f..6db0a720c 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -3,11 +3,11 @@ package types import ( "bytes" "context" - "crypto/sha256" "math/big" block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + "github.com/minio/sha256-simd" "github.com/multiformats/go-multihash" "go.opencensus.io/trace" xerrors "golang.org/x/xerrors" @@ -166,19 +166,24 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { /* Need to check that - h(vrfout) / 2^256 < minerPower / totalPower + h(vrfout) / max(h) * e < minerPower / totalPower + max(h) == 2^256-1 + which in terms of integer math means: + h(vrfout) * totalPower * e < minerPower * (2^256-1) */ h := sha256.Sum256(eproof) - // 2^256 - rden := BigInt{big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil)} + lhs := BigFromBytes(h[:]).Int + lhs = lhs.Mul(lhs, blocksPerEpoch.Int) + lhs = lhs.Mul(lhs, totpow.Int) - top := BigMul(BigMul(rden, mpow), blocksPerEpoch) - out := BigDiv(top, totpow) + // rhs = minerPower * 2^256 - minerPower + // rhs = minerPower << 256 - minerPower + rhs := new(big.Int).Lsh(mpow.Int, 256) + rhs = rhs.Sub(rhs, mpow.Int) - hp := BigFromBytes(h[:]) - return hp.LessThan(out) + return lhs.Cmp(rhs) == -1 } func (t *Ticket) Equals(ot *Ticket) bool { diff --git a/go.mod b/go.mod index 343315e1c..b8eec445f 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/mattn/go-runewidth v0.0.4 // indirect github.com/miekg/dns v1.1.16 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 + github.com/minio/sha256-simd v0.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.0.4 From e98feb4a391ffb983ae9b951aaf6a488279e88c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 28 Oct 2019 21:28:01 +0100 Subject: [PATCH 053/230] fix PowerCmp --- chain/types/blockheader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 6db0a720c..1ede605d4 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -166,21 +166,21 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { /* Need to check that - h(vrfout) / max(h) * e < minerPower / totalPower + h(vrfout) / max(h) < e * minerPower / totalPower max(h) == 2^256-1 which in terms of integer math means: - h(vrfout) * totalPower * e < minerPower * (2^256-1) + h(vrfout) * totalPower < e * minerPower * (2^256-1) */ h := sha256.Sum256(eproof) lhs := BigFromBytes(h[:]).Int - lhs = lhs.Mul(lhs, blocksPerEpoch.Int) lhs = lhs.Mul(lhs, totpow.Int) // rhs = minerPower * 2^256 - minerPower // rhs = minerPower << 256 - minerPower rhs := new(big.Int).Lsh(mpow.Int, 256) + rhs = rhs.Mul(rhs, blocksPerEpoch.Int) rhs = rhs.Sub(rhs, mpow.Int) return lhs.Cmp(rhs) == -1 From 72af55d0677819a3811cc43520a4ffb47230581e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 11:01:18 +0100 Subject: [PATCH 054/230] deals: Price per epoch --- api/api.go | 6 +++--- chain/actors/actor_storagemarket.go | 18 ++++++++++++------ chain/actors/cbor_gen.go | 8 ++++---- chain/deals/cbor_gen.go | 8 ++++---- chain/deals/client.go | 24 ++++++++++++------------ chain/deals/provider_states.go | 8 ++++---- chain/vm/vm.go | 1 - node/impl/client/client.go | 11 ++++------- 8 files changed, 43 insertions(+), 41 deletions(-) diff --git a/api/api.go b/api/api.go index 94f69485c..989d42ee8 100644 --- a/api/api.go +++ b/api/api.go @@ -102,7 +102,7 @@ type FullNode interface { // ClientImport imports file under the specified path into filestore ClientImport(ctx context.Context, path string) (cid.Cid, error) - ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) + ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error) ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) @@ -206,8 +206,8 @@ type DealInfo struct { PieceRef []byte // cid bytes Size uint64 - TotalPrice types.BigInt - Duration uint64 + PricePerEpoch types.BigInt + Duration uint64 } type MsgWait struct { diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 57f7171a5..9a5f88435 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -85,12 +85,16 @@ type StorageDealProposal struct { // Changing to duration makes sure that the price-per-block is defined, and the miner // doesn't get paid when not storing the sector - StoragePrice types.BigInt - StorageCollateral types.BigInt + StoragePricePerEpoch types.BigInt + StorageCollateral types.BigInt ProposerSignature *types.Signature } +func (sdp *StorageDealProposal) TotalStoragePrice() types.BigInt { + return types.BigMul(sdp.StoragePricePerEpoch, types.NewInt(sdp.Duration)) +} + type SignFunc = func(context.Context, []byte) (*types.Signature, error) func (sdp *StorageDealProposal) Sign(ctx context.Context, sign SignFunc) error { @@ -392,11 +396,13 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe clientBalance := b[0] providerBalance := b[1] - if clientBalance.Available.LessThan(deal.Proposal.StoragePrice) { - return aerrors.Newf(5, "client doesn't have enough available funds to cover StoragePrice; %d < %d", clientBalance.Available, deal.Proposal.StoragePrice) + totalPrice := deal.Proposal.TotalStoragePrice() + + if clientBalance.Available.LessThan(totalPrice) { + return aerrors.Newf(5, "client doesn't have enough available funds to cover storage price; %d < %d", clientBalance.Available, totalPrice) } - clientBalance = lockFunds(clientBalance, deal.Proposal.StoragePrice) + clientBalance = lockFunds(clientBalance, totalPrice) // TODO: REVIEW: Not clear who pays for this if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) { @@ -539,7 +545,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // todo: check math (written on a plane, also tired) // TODO: division is hard, this more than likely has some off-by-one issue - toPay := types.BigDiv(types.BigMul(dealInfo.Deal.Proposal.StoragePrice, types.NewInt(build.ProvingPeriodDuration)), types.NewInt(dealInfo.Deal.Proposal.Duration)) + toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.ProvingPeriodDuration)) b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) if aerr != nil { diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index b45586c96..5444dafb0 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3051,8 +3051,8 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.StoragePrice (types.BigInt) - if err := t.StoragePrice.MarshalCBOR(w); err != nil { + // t.t.StoragePricePerEpoch (types.BigInt) + if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { return err } @@ -3158,11 +3158,11 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.Duration = extra - // t.t.StoragePrice (types.BigInt) + // t.t.StoragePricePerEpoch (types.BigInt) { - if err := t.StoragePrice.UnmarshalCBOR(br); err != nil { + if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil { return err } diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 0e09cc492..40413170c 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -419,8 +419,8 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Data: %w", err) } - // t.t.TotalPrice (types.BigInt) - if err := t.TotalPrice.MarshalCBOR(w); err != nil { + // t.t.PricePerEpoch (types.BigInt) + if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { return err } @@ -481,11 +481,11 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { t.Data = c } - // t.t.TotalPrice (types.BigInt) + // t.t.PricePerEpoch (types.BigInt) { - if err := t.TotalPrice.UnmarshalCBOR(br); err != nil { + if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil { return err } diff --git a/chain/deals/client.go b/chain/deals/client.go index b6e0b3df5..fa70dbe99 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -159,7 +159,7 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { type ClientDealProposal struct { Data cid.Cid - TotalPrice types.BigInt + PricePerEpoch types.BigInt ProposalExpiration uint64 Duration uint64 @@ -175,13 +175,13 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("getting client market balance failed: %w", err) } - if clientMarketBalance.Available.LessThan(p.TotalPrice) { + if clientMarketBalance.Available.LessThan(types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))) { // TODO: move to a smarter market funds manager smsg, err := c.mpool.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: p.Client, - Value: p.TotalPrice, + Value: types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)), GasPrice: types.NewInt(0), GasLimit: types.NewInt(1000000), Method: actors.SMAMethods.AddBalance, @@ -203,15 +203,15 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro dataSize, err := c.dataSize(ctx, p.Data) proposal := &actors.StorageDealProposal{ - PieceRef: p.Data.Bytes(), - PieceSize: uint64(dataSize), - PieceSerialization: actors.SerializationUnixFSv0, - Client: p.Client, - Provider: p.ProviderAddress, - ProposalExpiration: p.ProposalExpiration, - Duration: p.Duration, - StoragePrice: p.TotalPrice, - StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc + PieceRef: p.Data.Bytes(), + PieceSize: uint64(dataSize), + PieceSerialization: actors.SerializationUnixFSv0, + Client: p.Client, + Provider: p.ProviderAddress, + ProposalExpiration: p.ProposalExpiration, + Duration: p.Duration, + StoragePricePerEpoch: p.PricePerEpoch, + StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc } if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil { diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index c2f49d4f0..72e7d17bf 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -88,9 +88,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral // TODO: - minPrice := types.BigMul(p.ask.Ask.Price, types.BigMul(types.NewInt(deal.Proposal.Duration), types.NewInt(deal.Proposal.PieceSize))) - if deal.Proposal.StoragePrice.LessThan(minPrice) { - return nil, xerrors.Errorf("storage price less than asking price: %s < %s", deal.Proposal.StoragePrice, minPrice) + minPrice := types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)) + if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { + return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) } if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize { @@ -105,7 +105,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // This doesn't guarantee that the client won't withdraw / lock those funds // but it's a decent first filter - if clientMarketBalance.Available.LessThan(deal.Proposal.StoragePrice) { + if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) { return nil, xerrors.New("clientMarketBalance.Available too small") } diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 8b702c48b..38831043b 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -488,7 +488,6 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet, return nil, xerrors.Errorf("getting block miner actor (%s) failed: %w", vm.blockMiner, err) } - // TODO: support multiple blocks in a tipset // TODO: actually wire this up (miner is undef for now) gasReward := types.BigMul(msg.GasPrice, gasUsed) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 926d6f623..a785bef44 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -53,7 +53,7 @@ type API struct { Filestore dtypes.ClientFilestore `optional:"true"` } -func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { +func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error) { // TODO: make this a param self, err := a.WalletDefaultAddress(ctx) if err != nil { @@ -76,12 +76,9 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A return nil, err } - // setup payments - total := types.BigMul(price, types.NewInt(blocksDuration)) - proposal := deals.ClientDealProposal{ Data: data, - TotalPrice: total, + PricePerEpoch: epochPrice, ProposalExpiration: math.MaxUint64, // TODO: set something reasonable Duration: blocksDuration, ProviderAddress: miner, @@ -110,8 +107,8 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { PieceRef: v.Proposal.PieceRef, Size: v.Proposal.PieceSize, - TotalPrice: v.Proposal.StoragePrice, - Duration: v.Proposal.Duration, + PricePerEpoch: v.Proposal.StoragePricePerEpoch, + Duration: v.Proposal.Duration, } } From 738e8c5a3c1eb9cc1a186487907c83ffbecf5648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 11:19:39 +0100 Subject: [PATCH 055/230] deals: Ask prices per GiB --- chain/deals/provider.go | 2 +- chain/deals/provider_states.go | 2 +- chain/types/ask.go | 2 ++ cli/client.go | 5 +++-- lotuspond/front/src/Client.js | 4 ++-- lotuspond/front/src/State.js | 4 ++-- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 5a3c0ab65..86383c61b 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -109,7 +109,7 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt * if h.ask == nil { // TODO: we should be fine with this state, and just say it means 'not actively accepting deals' // for now... lets just set a price - if err := h.SetPrice(types.NewInt(3), 1000000); err != nil { + if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil { return nil, xerrors.Errorf("failed setting a default price: %w", err) } } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 72e7d17bf..7df6c29c6 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -88,7 +88,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral // TODO: - minPrice := types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)) + minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1 << 30)) if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) } diff --git a/chain/types/ask.go b/chain/types/ask.go index 1e74c9cb6..3e1767b06 100644 --- a/chain/types/ask.go +++ b/chain/types/ask.go @@ -16,7 +16,9 @@ type SignedStorageAsk struct { } type StorageAsk struct { + // Price per GiB / Epoch Price BigInt + MinPieceSize uint64 Miner address.Address Timestamp uint64 diff --git a/cli/client.go b/cli/client.go index e9e5814c7..2a6b855bf 100644 --- a/cli/client.go +++ b/cli/client.go @@ -314,13 +314,14 @@ var clientQueryAskCmd = &cli.Command{ if size == 0 { return nil } - fmt.Printf("Price per Block: %s\n", types.BigMul(ask.Ask.Price, types.NewInt(uint64(size)))) + perEpoch := types.BigDiv(types.BigMul(ask.Ask.Price, types.NewInt(uint64(size))), types.NewInt(1<<30)) + fmt.Printf("Price per Block: %s\n", perEpoch) duration := cctx.Int64("duration") if duration == 0 { return nil } - fmt.Printf("Total Price: %s\n", types.BigMul(types.BigMul(ask.Ask.Price, types.NewInt(uint64(size))), types.NewInt(uint64(duration)))) + fmt.Printf("Total Price: %s\n", types.BigMul(perEpoch, types.NewInt(uint64(duration)))) return nil }, diff --git a/lotuspond/front/src/Client.js b/lotuspond/front/src/Client.js index 35aaab148..37606223b 100644 --- a/lotuspond/front/src/Client.js +++ b/lotuspond/front/src/Client.js @@ -21,7 +21,7 @@ class Client extends React.Component { this.state = { miners: ["t0101"], - ask: {Price: "3"}, + ask: {Price: "500000000"}, kbs: 1, blocks: 12, @@ -52,7 +52,7 @@ class Client extends React.Component { update = (name) => (e) => this.setState({ [name]: e.target.value }); makeDeal = async () => { - let perBlk = this.state.ask.Price * this.state.kbs * 1000 + let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) let file = await this.props.pondClient.call('Pond.CreateRandomFile', [this.state.kbs * 1000]) // 1024 won't fit in 1k blocks :( let cid = await this.props.client.call('Filecoin.ClientImport', [file]) diff --git a/lotuspond/front/src/State.js b/lotuspond/front/src/State.js index 6fb1a70aa..93252505d 100644 --- a/lotuspond/front/src/State.js +++ b/lotuspond/front/src/State.js @@ -123,10 +123,10 @@ class MarketState extends React.Component { {Object.keys(this.state.deals).map(d => {d} {this.state.deals[d].ActivationEpoch || "No"} -
+
{this.state.deals[d].Deal.Proposal.PieceSize}B - {this.state.deals[d].Deal.Proposal.StoragePrice} + {this.state.deals[d].Deal.Proposal.StoragePricePerEpoch*this.state.deals[d].Deal.Proposal.Duration} {this.state.deals[d].Deal.Proposal.Duration} )} From bee0ecf97fc7cd6d4733ba00286b8836bafe8404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:02:13 +0100 Subject: [PATCH 056/230] deals: Nicer output in client cli --- chain/actors/actor_storagemarket.go | 8 +------- cli/client.go | 13 ++++++------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 9a5f88435..9c3fcb9aa 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -77,13 +77,7 @@ type StorageDealProposal struct { Provider address.Address ProposalExpiration uint64 - Duration uint64 // TODO: spec proposes 'DealExpiration', but that's awkward as it - // doesn't tell when the deal actually starts, so the price per block is impossible to - // calculate. It also doesn't incentivize the miner to seal / activate sooner, as he - // still get's paid the full amount specified in the deal - // - // Changing to duration makes sure that the price-per-block is defined, and the miner - // doesn't get paid when not storing the sector + Duration uint64 // TODO: spec StoragePricePerEpoch types.BigInt StorageCollateral types.BigInt diff --git a/cli/client.go b/cli/client.go index 2a6b855bf..7f9771a3d 100644 --- a/cli/client.go +++ b/cli/client.go @@ -101,8 +101,7 @@ var clientDealCmd = &cli.Command{ return err } - // TODO: parse bigint - price, err := strconv.ParseInt(cctx.Args().Get(2), 10, 32) + price, err := types.ParseFIL(cctx.Args().Get(2)) if err != nil { return err } @@ -112,7 +111,7 @@ var clientDealCmd = &cli.Command{ return err } - proposal, err := api.ClientStartDeal(ctx, data, miner, types.NewInt(uint64(price)), uint64(dur)) + proposal, err := api.ClientStartDeal(ctx, data, miner, types.BigInt(price), uint64(dur)) if err != nil { return err } @@ -164,7 +163,7 @@ var clientFindCmd = &cli.Command{ fmt.Printf("ERR %s@%s: %s\n", offer.Miner, offer.MinerPeerID, offer.Err) continue } - fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, offer.MinPrice, offer.Size) + fmt.Printf("RETRIEVAL %s@%s-%sfil-%db\n", offer.Miner, offer.MinerPeerID, types.FIL(offer.MinPrice), offer.Size) } return nil @@ -308,20 +307,20 @@ var clientQueryAskCmd = &cli.Command{ } fmt.Printf("Ask: %s\n", maddr) - fmt.Printf("Price per Byte: %s\n", ask.Ask.Price) + fmt.Printf("Price per Byte: %s\n", types.FIL(ask.Ask.Price)) size := cctx.Int64("size") if size == 0 { return nil } perEpoch := types.BigDiv(types.BigMul(ask.Ask.Price, types.NewInt(uint64(size))), types.NewInt(1<<30)) - fmt.Printf("Price per Block: %s\n", perEpoch) + fmt.Printf("Price per Block: %s\n", types.FIL(perEpoch)) duration := cctx.Int64("duration") if duration == 0 { return nil } - fmt.Printf("Total Price: %s\n", types.BigMul(perEpoch, types.NewInt(uint64(duration)))) + fmt.Printf("Total Price: %s\n", types.FIL(types.BigMul(perEpoch, types.NewInt(uint64(duration))))) return nil }, From be4a5665fe480f28c95d848e60a3bff55735b48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:02:24 +0100 Subject: [PATCH 057/230] deals: gofmt --- chain/deals/provider_states.go | 2 +- chain/types/ask.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 7df6c29c6..595f236da 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -88,7 +88,7 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral // TODO: - minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1 << 30)) + minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) } diff --git a/chain/types/ask.go b/chain/types/ask.go index 3e1767b06..dd768feb3 100644 --- a/chain/types/ask.go +++ b/chain/types/ask.go @@ -17,7 +17,7 @@ type SignedStorageAsk struct { type StorageAsk struct { // Price per GiB / Epoch - Price BigInt + Price BigInt MinPieceSize uint64 Miner address.Address From 774bc2f8ec614420f818406da837f85903e0484b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:47:19 +0100 Subject: [PATCH 058/230] scripts: Simple guard in deploy-devnet.sh --- scripts/deploy-devnet.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/deploy-devnet.sh b/scripts/deploy-devnet.sh index f0c2f6809..eff319626 100755 --- a/scripts/deploy-devnet.sh +++ b/scripts/deploy-devnet.sh @@ -7,6 +7,13 @@ BOOTSTRAPPERS=( root@147.75.80.17 ) ############ +read -p "You are about to deploy new DevNet, killing bootstrap nodes. Proceed? (y/n)?" r +case "$r" in + y|Y ) echo "Proceding";; + n|N ) exit 0;; + * ) exit 1;; +esac + log() { echo -e "\e[33m$1\e[39m" } From 5305ef68711fe27e3eb7686a99ec91c3b48a64b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:54:26 +0100 Subject: [PATCH 059/230] wallet: Print new line in import cmd --- cli/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/wallet.go b/cli/wallet.go index cb0cbbdd3..7ee4eab16 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -231,7 +231,7 @@ var walletImport = &cli.Command{ return err } - fmt.Printf("imported key %s successfully!", addr) + fmt.Printf("imported key %s successfully!\n", addr) return nil }, } From b179abceb346259a2e272562390ff51a1e0975a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:55:33 +0100 Subject: [PATCH 060/230] scripts: Fix ./lotus call in deploy-devnet.sh --- scripts/deploy-devnet.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deploy-devnet.sh b/scripts/deploy-devnet.sh index eff319626..ac5b800d1 100755 --- a/scripts/deploy-devnet.sh +++ b/scripts/deploy-devnet.sh @@ -7,7 +7,7 @@ BOOTSTRAPPERS=( root@147.75.80.17 ) ############ -read -p "You are about to deploy new DevNet, killing bootstrap nodes. Proceed? (y/n)?" r +read -p "You are about to deploy new DevNet, killing bootstrap nodes. Proceed? (y/n)? " r case "$r" in y|Y ) echo "Proceding";; n|N ) exit 0;; @@ -81,7 +81,7 @@ ssh $GENESIS_HOST 'systemctl start lotus-storage-miner' log 'Getting genesis addr info' -ssh $GENESIS_HOST './lotus net listen' | grep -v '/10' | grep -v '/127' > build/bootstrap/root.pi +ssh $GENESIS_HOST 'lotus net listen' | grep -v '/10' | grep -v '/127' > build/bootstrap/root.pi log '> Creating bootstrap binaries' make @@ -113,5 +113,5 @@ do log 'Extracting addr info' - ssh "$host" './lotus net listen' | grep -v '/10' | grep -v '/127' >> build/bootstrap/bootstrappers.pi + ssh "$host" 'lotus net listen' | grep -v '/10' | grep -v '/127' >> build/bootstrap/bootstrappers.pi done From f6ba2f0b9df8f743818b1b320595276096bf0ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 13:56:20 +0100 Subject: [PATCH 061/230] Devnet 6 --- build/bootstrap/bootstrappers.pi | 2 +- build/bootstrap/root.pi | 2 +- build/genesis/devnet.car | Bin 1844 -> 2218 bytes 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 031804bbd..0319a6d97 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1 +1 @@ -/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWQ3NBPgTbXFRGnmpZUpYBSPJtwJfxuAFHBuWJ8LumCYf2 +/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWPTYfTi7B1mQpbyRaCT65Trf7zW9npmihcF1xPfZogrzh diff --git a/build/bootstrap/root.pi b/build/bootstrap/root.pi index f1f0ea3a5..b03183ddf 100644 --- a/build/bootstrap/root.pi +++ b/build/bootstrap/root.pi @@ -1 +1 @@ -/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWGU8C1mFsEtz4bXmHUH3kQTnQnxVy8cigwGV94qCpYJw7 \ No newline at end of file +/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWEwGkxuG4BSvWUU8Sf69rxUYjWHAMT2qFVBhzGJZAXc6v diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index a629514c1731cf1f4fc4e0cc8a08ffd15bd0b995..ea19dc0d723bf81901302a36094dfa94e20d1c75 100644 GIT binary patch literal 2218 zcmcCnlvMCPjjoP`Aq$ZTb`btfTWc1?4>;ovC;U#8;B^t{B9(xOx#kc!=5rWwoxcAWyU#0M5gU=VU-Q-quY10w?z zFbXgTIKbEtx_x1yVPjJg14GLVEq?~aPzJ8F%p83~{mi_~5+r9gOq%Yj$*|ChC%r61 zwVvt49v%mU{@FG5JDMzS-n_07&)@(xCmCo?5Xc-}h&hSL$@!&u$VOexQj=U(GMH`FF);4JtQ`<{Ow7{y#Z@;;6^c0Zj>(rTba9$%}sf|n+*&MP$#AW%?$uK zksIQ~+|0bxA|$_R=I{KP605i)@~q*!kG{)hB^Q_Z$oJ)bR=O`OU?9YHmeHGmF(G!| z%QkfZxS7mAGXp_p@<7c@EXqzr_O;RR*{3@U?|S^0mD#j(R#Et(6`q@dV!u_!FysjH zeC@M<`#ObUUjyAId}gV}_ao;E;O=8W^K(IdIf|dN0)M&A(K$(LOPYZ9%T*crZftofDsU!rF5(%Y{CZrISxqRq#qN?KQkIGs>BUm%5+1d&qXcBuCWCB+`3K;0+sn%8Lkk!iYH zQ#QqRWr&~ZP`Nf=bIHE`eSW3?L#F<9a6nQ_LWxC4sUrgjIx`4&I5;rOfjXR!V_G6e z4|p=PWEP~R7AbhTC>TnC3PD{+%n z5JL#%ePln_JFkg3QJH#aGm~;Nf8)O=PTJDDgB`Y=a@*1uJV)d90Y*?daDkXhNNrOP bu>3>=jj%GBu@+(oq52ZJSRoW@Bm^)3z0Px1A(3l}!- zIKO_;lk~FGqTY+AO4METaI3MxWsggqjS=KhGxSBHfe%;=c{C`i+GBpl=1AO`a5SUEq?7X zvNL+l;}Rm>)%^9&6xj)Fsj2UyK6)W3?ak-oeBf%e;a6@AS_lFsQv#UgHv5 z@j>C>DF`5C#FEmYR3MOwUA{*smGbS862P{Bm+(G0hz#_n4Fwnnul!2m>rf^uof~GG9V3>hG6F43!44J3w^NY%VcE>~g+C+V~rzxyKANu_Z5;V8J zSgrBQYPr^~AMJV~@iQkztvc|$PA6sRW{DKA&Qc~V&sY6o)5SNshVm|Ch^lz;rQ+Tv ziBut^lm<<3>W}~+l-L#sAW{P)O>BpHl2C%^K?=I$Y^4)LZ;lG9H5@mN<=dF_?ZBd} z{@j Date: Tue, 29 Oct 2019 14:06:02 +0100 Subject: [PATCH 062/230] townhall: Don't hard-code stuff --- Makefile | 2 +- cmd/lotus-townhall/main.go | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6c3a6049f..b30f8712f 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ townhall: rm -f townhall go build -o townhall ./cmd/lotus-townhall (cd ./cmd/lotus-townhall/townhall && npm i && npm run build) - go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall + go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build .PHONY: townhall fountain: diff --git a/cmd/lotus-townhall/main.go b/cmd/lotus-townhall/main.go index 9be633147..e45b8e8e4 100644 --- a/cmd/lotus-townhall/main.go +++ b/cmd/lotus-townhall/main.go @@ -1,9 +1,14 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/build" + "github.com/ipfs/go-car" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" "net/http" "strings" @@ -14,11 +19,26 @@ import ( pnet "github.com/libp2p/go-libp2p-pnet" pubsub "github.com/libp2p/go-libp2p-pubsub" - "github.com/filecoin-project/lotus/lib/addrutil" "github.com/filecoin-project/lotus/node/modules/lp2p" ) -const topic = "/fil/headnotifs/bafy2bzacea77zxnepp7wuqqgpj7xcw2ywwmmcmtrbjghhv4g2dildogpv6roi" +var topic = "/fil/headnotifs/" + +func init() { + genBytes := build.MaybeGenesis() + bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + + c, err := car.LoadCar(bs, bytes.NewReader(genBytes)) + if err != nil { + panic(err) + } + if len(c.Roots) != 1 { + panic("expected genesis file to have one root") + } + + fmt.Printf("Genesis CID: %s\n", c.Roots[0]) + topic = topic + c.Roots[0].String() +} var upgrader = websocket.Upgrader{ WriteBufferSize: 1024, @@ -48,9 +68,7 @@ func main() { panic(err) } - pi, err := addrutil.ParseAddresses(ctx, []string{ - "/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWGU8C1mFsEtz4bXmHUH3kQTnQnxVy8cigwGV94qCpYJw7", - }) + pi, err := build.BuiltinBootstrap() if err != nil { panic(err) } From 23b7cf305f4070c77fa78f8c9a8dfe3735406ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 14:10:53 +0100 Subject: [PATCH 063/230] fountain: Default to 1GiB sectors --- cmd/lotus-fountain/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 705c7d148..3e2e0a772 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -254,7 +254,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { params, err := actors.SerializeParams(&actors.CreateStorageMinerParams{ Owner: owner, Worker: owner, - SectorSize: build.SectorSizes[0], // TODO: dropdown allowing selection + SectorSize: 1 << 30, // build.SectorSizes[0], // TODO: dropdown allowing selection (1GiB for now) PeerID: peer.ID("SETME"), }) if err != nil { From 640b889aa61b975a76580befeebee0bd41980b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 14:26:15 +0100 Subject: [PATCH 064/230] fountain: Fix create button in /miner.html --- cmd/lotus-fountain/site/miner.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-fountain/site/miner.html b/cmd/lotus-fountain/site/miner.html index 31952269e..8aec74eac 100644 --- a/cmd/lotus-fountain/site/miner.html +++ b/cmd/lotus-fountain/site/miner.html @@ -14,7 +14,7 @@
Enter destination address: - +
-
+ Enter destination address: +
From 169c285fb3335ed1c66575bff4987bf75031e87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 20:46:32 +0100 Subject: [PATCH 069/230] storageminer: Handle uncommited sectors on start --- lib/sectorbuilder/sectorbuilder.go | 4 ++++ storage/commitment/tracker.go | 9 +++++++ storage/miner.go | 38 ++++++++++++++++++++++++++++++ storage/sector/store.go | 4 ++++ 4 files changed, 55 insertions(+) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index c9cad6ae5..26123e757 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -110,6 +110,10 @@ func (sb *SectorBuilder) GetAllStagedSectors() ([]uint64, error) { return out, nil } +func (sb *SectorBuilder) GetAllSealedSectors() ([]SealedSectorMetadata, error) { + return sectorbuilder.GetAllSealedSectors(sb.handle) +} + func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]byte, error) { // Wait, this is a blocking method with no way of interrupting it? // does it checkpoint itself? diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go index aa50c705e..10510fba0 100644 --- a/storage/commitment/tracker.go +++ b/storage/commitment/tracker.go @@ -136,3 +136,12 @@ func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sector return cid.Undef, ctx.Err() } } + +func (ct *Tracker) CheckCommitment(miner address.Address, sectorId uint64) (bool, error) { + key := commitmentKey(miner, sectorId) + + ct.lk.Lock() + defer ct.lk.Unlock() + + return ct.commitments.Has(key) +} diff --git a/storage/miner.go b/storage/miner.go index 0ab1ce009..81880f86b 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/go-sectorbuilder/sealing_state" "sync" "github.com/ipfs/go-cid" @@ -93,10 +94,47 @@ func (m *Miner) Run(ctx context.Context) error { return nil } +func (m *Miner) commitUntrackedSectors(ctx context.Context) error { + sealed, err := m.secst.Sealed() + if err != nil { + return err + } + + for _, s := range sealed { + has, err := m.commt.CheckCommitment(m.maddr, s.SectorID) + if err != nil { + log.Error("checking commitment: ", err) + } + + if has { + continue + } + + log.Warnf("Missing commitment for sector %d, committing sector", s.SectorID) + + if err := m.commitSector(ctx, sectorbuilder.SectorSealingStatus{ + SectorID: s.SectorID, + State: sealing_state.Sealed, + CommD: s.CommD, + CommR: s.CommR, + Proof: s.Proof, + Pieces: s.Pieces, + Ticket: s.Ticket, + }); err != nil { + log.Error("Committing uncommitted sector failed: ", err) + } + } + return nil +} + func (m *Miner) handlePostingSealedSectors(ctx context.Context) { incoming := m.secst.Incoming() defer m.secst.CloseIncoming(incoming) + if err := m.commitUntrackedSectors(ctx); err != nil { + log.Error(err) + } + for { select { case sinfo, ok := <-incoming: diff --git a/storage/sector/store.go b/storage/sector/store.go index 5a4b6a9a3..f82505837 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -263,6 +263,10 @@ func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.Sect return s.sb.SealStatus(sector) } +func (s *Store) Sealed() ([]sectorbuilder.SealedSectorMetadata, error) { + return s.sb.GetAllSealedSectors() +} + func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte, faults []uint64) ([]byte, error) { sbsi := make([]sectorbuilder.SectorInfo, len(sectors)) for k, sector := range sectors { From 391c73774dec1b1144c17689723d1310345cae36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 20:51:44 +0100 Subject: [PATCH 070/230] Set swarm log level to WARN --- cmd/lotus-storage-miner/main.go | 2 ++ cmd/lotus/main.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index c9a904695..f8bf2aac1 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -18,6 +18,8 @@ const FlagStorageRepo = "storagerepo" func main() { logging.SetLogLevel("*", "INFO") + logging.SetLogLevel("swarm", "WARN") + local := []*cli.Command{ runCmd, initCmd, diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index e7cffe186..38fc478ac 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -16,6 +16,8 @@ import ( func main() { logging.SetLogLevel("*", "INFO") logging.SetLogLevel("dht", "ERROR") + logging.SetLogLevel("swarm", "WARN") + local := []*cli.Command{ DaemonCmd, } From 44f4ee0de1c628c330abc74a8328dedc9b1b7a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 21:41:30 +0100 Subject: [PATCH 071/230] storageminer: cmd to list sector commitments --- api/api.go | 10 +++++ api/struct.go | 6 +++ cmd/lotus-storage-miner/commitments.go | 41 ++++++++++++++++++ cmd/lotus-storage-miner/main.go | 1 + node/impl/storminer.go | 6 +++ storage/commitment/tracker.go | 57 ++++++++++++++++++++++++++ storage/miner.go | 2 +- 7 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 cmd/lotus-storage-miner/commitments.go diff --git a/api/api.go b/api/api.go index 635901ecb..463e5e9cb 100644 --- a/api/api.go +++ b/api/api.go @@ -168,6 +168,8 @@ type StorageMiner interface { SectorsList(context.Context) ([]uint64, error) SectorsRefs(context.Context) (map[string][]SealedRef, error) + + CommitmentsList(context.Context) ([]SectorCommitment, error) } // Version provides various build-time information @@ -330,6 +332,14 @@ type SyncState struct { Height uint64 } +type SectorCommitment struct { + SectorID uint64 + Miner address.Address + + CommitMsg cid.Cid + DealIDs []uint64 +} + type SyncStateStage int const ( diff --git a/api/struct.go b/api/struct.go index 6eefe41dc..47d88aa69 100644 --- a/api/struct.go +++ b/api/struct.go @@ -133,6 +133,8 @@ type StorageMinerStruct struct { SectorsList func(context.Context) ([]uint64, error) `perm:"read"` SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` + + CommitmentsList func(context.Context) ([]SectorCommitment, error) `perm:"read"` } } @@ -479,6 +481,10 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]Seal return c.Internal.SectorsRefs(ctx) } +func (c *StorageMinerStruct) CommitmentsList(ctx context.Context) ([]SectorCommitment, error) { + return c.Internal.CommitmentsList(ctx) +} + var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/cmd/lotus-storage-miner/commitments.go b/cmd/lotus-storage-miner/commitments.go new file mode 100644 index 000000000..a7825deb1 --- /dev/null +++ b/cmd/lotus-storage-miner/commitments.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + lcli "github.com/filecoin-project/lotus/cli" + + "gopkg.in/urfave/cli.v2" +) + +var commitmentsCmd = &cli.Command{ + Name: "commitments", + Usage: "interact with commitment tracker", + Subcommands: []*cli.Command{ + commitmentsListCmd, + }, +} + +var commitmentsListCmd = &cli.Command{ + Name: "list", + Usage: "List tracked sector commitments", + Action: func(cctx *cli.Context) error { + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + comms, err := api.CommitmentsList(ctx) + if err != nil { + return err + } + + for _, comm := range comms { + fmt.Printf("%s:%d msg:%s, deals: %v\n", comm.Miner, comm.SectorID, comm.CommitMsg, comm.DealIDs) + } + + return nil + }, +} diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index c9a904695..c08cd21e6 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -24,6 +24,7 @@ func main() { infoCmd, storeGarbageCmd, sectorsCmd, + commitmentsCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/node/impl/storminer.go b/node/impl/storminer.go index c763dab5e..8eb363fe2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -23,6 +24,7 @@ type StorageMinerAPI struct { SectorBuilder *sectorbuilder.SectorBuilder Sectors *sector.Store SectorBlocks *sectorblocks.SectorBlocks + CommitmentTracker *commitment.Tracker Miner *storage.Miner } @@ -81,4 +83,8 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed return out, nil } +func (sm *StorageMinerAPI) CommitmentsList(ctx context.Context) ([]api.SectorCommitment, error) { + return sm.CommitmentTracker.List() +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go index 10510fba0..a0fa57e80 100644 --- a/storage/commitment/tracker.go +++ b/storage/commitment/tracker.go @@ -3,6 +3,8 @@ package commitment import ( "context" "fmt" + "strconv" + "strings" "sync" "github.com/ipfs/go-cid" @@ -12,8 +14,10 @@ import ( logging "github.com/ipfs/go-log" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/node/modules/dtypes" + dsq "github.com/ipfs/go-datastore/query" ) var log = logging.Logger("commitment") @@ -145,3 +149,56 @@ func (ct *Tracker) CheckCommitment(miner address.Address, sectorId uint64) (bool return ct.commitments.Has(key) } + +func (ct *Tracker) List() ([]api.SectorCommitment, error) { + out := make([]api.SectorCommitment, 0) + + ct.lk.Lock() + defer ct.lk.Unlock() + + res, err := ct.commitments.Query(dsq.Query{}) + if err != nil { + return nil, err + } + defer res.Close() + + for { + res, ok := res.NextSync() + if !ok { + break + } + + if res.Error != nil { + return nil, xerrors.Errorf("iterating commitments: %w", err) + } + + parts := strings.Split(res.Key, "/") + if len(parts) != 4 { + return nil, xerrors.Errorf("expected commitment key to be 4 parts, Key %s", res.Key) + } + + miner, err := address.NewFromString(parts[2]) + if err != nil { + return nil, xerrors.Errorf("parsing miner address: %w", err) + } + + sectorID, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil, xerrors.Errorf("parsing sector id: %w", err) + } + + var comm commitment + if err := cbor.DecodeInto(res.Value, &comm); err != nil { + return nil, xerrors.Errorf("decoding commitment %s (`% X`): %w", res.Key, res.Value, err) + } + + out = append(out, api.SectorCommitment{ + SectorID: uint64(sectorID), + Miner: miner, + CommitMsg: comm.Msg, + DealIDs: comm.DealIDs, + }) + } + + return out, nil +} diff --git a/storage/miner.go b/storage/miner.go index 81880f86b..7e1757e31 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,9 +2,9 @@ package storage import ( "context" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" "sync" + "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" From 3047c9d4c2367f830b213611f7d359071376b1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 22:41:28 +0100 Subject: [PATCH 072/230] storageminer: Check on-chain commitments --- storage/commitment/tracker.go | 9 --------- storage/miner.go | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go index a0fa57e80..4e439c57d 100644 --- a/storage/commitment/tracker.go +++ b/storage/commitment/tracker.go @@ -141,15 +141,6 @@ func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sector } } -func (ct *Tracker) CheckCommitment(miner address.Address, sectorId uint64) (bool, error) { - key := commitmentKey(miner, sectorId) - - ct.lk.Lock() - defer ct.lk.Unlock() - - return ct.commitments.Has(key) -} - func (ct *Tracker) List() ([]api.SectorCommitment, error) { out := make([]api.SectorCommitment, 0) diff --git a/storage/miner.go b/storage/miner.go index 7e1757e31..ef045eb93 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -54,6 +54,7 @@ type storageMinerApi interface { StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) @@ -100,13 +101,18 @@ func (m *Miner) commitUntrackedSectors(ctx context.Context) error { return err } - for _, s := range sealed { - has, err := m.commt.CheckCommitment(m.maddr, s.SectorID) - if err != nil { - log.Error("checking commitment: ", err) - } + chainSectors, err := m.api.StateMinerSectors(ctx, m.maddr, nil) + if err != nil { + return err + } - if has { + onchain := map[uint64]struct{}{} + for _, chainSector := range chainSectors { + onchain[chainSector.SectorID] = struct{}{} + } + + for _, s := range sealed { + if _, ok := onchain[s.SectorID]; ok { continue } From b7a78670abb7feb52efd151b82a04d8964f43e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 22:58:41 +0100 Subject: [PATCH 073/230] storageminer: Call beginPosting before trying to commit a sector --- storage/miner.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/miner.go b/storage/miner.go index ef045eb93..4d16315f4 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -213,10 +213,6 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal return errors.Wrap(err, "pushing message to mpool") } - if err := m.commt.TrackCommitSectorMsg(m.maddr, sinfo.SectorID, smsg.Cid()); err != nil { - return errors.Wrap(err, "tracking sector commitment") - } - go func() { _, err := m.api.StateWaitMsg(ctx, smsg.Cid()) if err != nil { @@ -226,6 +222,10 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal m.beginPosting(ctx) }() + if err := m.commt.TrackCommitSectorMsg(m.maddr, sinfo.SectorID, smsg.Cid()); err != nil { + return xerrors.Errorf("tracking sector commitment: %w", err) + } + return nil } From 6c25c5384dc16ae55a083105d1050283cb076087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 23:04:08 +0100 Subject: [PATCH 074/230] commitment tracker: prefer new commitments --- storage/commitment/tracker.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go index 4e439c57d..90e2459bf 100644 --- a/storage/commitment/tracker.go +++ b/storage/commitment/tracker.go @@ -60,6 +60,21 @@ func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, tracking, err := ct.commitments.Get(key) switch err { + case nil: + var comm commitment + if err := cbor.DecodeInto(tracking, &comm); err != nil { + return err + } + + if !comm.Msg.Equals(commitMsg) { + log.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg) + } + + log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg) + + // we still want to store it + fallthrough // TODO: ideally we'd keep around both (even though we'll + // usually only need the new one) case datastore.ErrNotFound: comm := &commitment{Msg: commitMsg} commB, err := cbor.DumpObject(comm) @@ -77,18 +92,6 @@ func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, delete(ct.waits, key) } return nil - case nil: - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return err - } - - if !comm.Msg.Equals(commitMsg) { - return xerrors.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg) - } - - log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg) - return nil default: return err } From 922d8a90a5f5633a95f6419a386e025c1ffb083d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 29 Oct 2019 23:19:58 +0100 Subject: [PATCH 075/230] storageminer: Restart sealing on restart --- lib/sectorbuilder/sectorbuilder.go | 4 ++++ storage/sector/store.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 26123e757..0f7e2ea26 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -88,6 +88,10 @@ func (sb *SectorBuilder) SealSector(sectorID uint64, ticket SealTicket) (SealedS return sectorbuilder.SealSector(sb.handle, sectorID, ticket) } +func (sb *SectorBuilder) ResumeSealSector(sectorID uint64) (SealedSectorMetadata, error) { + return sectorbuilder.ResumeSealSector(sb.handle, sectorID) +} + func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { return sectorbuilder.GetSectorSealingStatusByID(sb.handle, sector) } diff --git a/storage/sector/store.go b/storage/sector/store.go index f82505837..5692c8649 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -107,9 +107,39 @@ func (s *Store) poll() { s.waitingLk.Unlock() } +func (s *Store) restartSealing() { + sectors, err := s.sb.GetAllStagedSectors() + if err != nil { + return + } + + for _, sid := range sectors { + status, err := s.sb.SealStatus(sid) + if err != nil { + return + } + + if status.State != sealing_state.Paused { + continue + } + + log.Infof("Sector %d is in paused state, resuming sealing", sid) + go func() { + // TODO: when we refactor wait-for-seal below, care about this output too + // (see SealSector below) + _, err := s.sb.ResumeSealSector(sid) + if err != nil { + return + } + }() + } +} + func (s *Store) service() { poll := time.Tick(5 * time.Second) + s.restartSealing() + for { select { case <-poll: From bc97a73d670359246c8013abffd233b57b63a35f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 30 Oct 2019 01:52:24 +0100 Subject: [PATCH 076/230] Implment formatter on FIL to override big.Int License: MIT Signed-off-by: Jakub Sztandera --- chain/types/fil.go | 9 +++++++++ cli/state.go | 2 +- cli/wallet.go | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/chain/types/fil.go b/chain/types/fil.go index c882c1852..f44336fb5 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -15,6 +15,15 @@ func (f FIL) String() string { return strings.TrimRight(r.FloatString(18), "0.") } +func (f FIL) Format(s fmt.State, ch rune) { + switch ch { + case 's', 'v': + fmt.Fprint(s, f.String()) + default: + f.Int.Format(s, ch) + } +} + func ParseFIL(s string) (FIL, error) { r, ok := new(big.Rat).SetString(s) if !ok { diff --git a/cli/state.go b/cli/state.go index 6d831b249..1448703f6 100644 --- a/cli/state.go +++ b/cli/state.go @@ -197,7 +197,7 @@ var statePledgeCollateralCmd = &cli.Command{ return err } - fmt.Println(types.FIL(coll).String()) + fmt.Println(types.FIL(coll)) return nil }, } diff --git a/cli/wallet.go b/cli/wallet.go index 7ee4eab16..a268056b9 100644 --- a/cli/wallet.go +++ b/cli/wallet.go @@ -105,7 +105,7 @@ var walletBalance = &cli.Command{ return err } - fmt.Printf("%s\n", types.FIL(balance).String()) + fmt.Printf("%s\n", types.FIL(balance)) return nil }, } From 5a5c66600fff68060b1af3b413df8a0ae2fec611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Oct 2019 10:55:49 +0100 Subject: [PATCH 077/230] storageminer: More correct listing of sealed sectors --- lib/sectorbuilder/sectorbuilder.go | 4 ---- storage/miner.go | 11 +---------- storage/sector/store.go | 22 ++++++++++++++++++++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 0f7e2ea26..a2e4499b7 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -114,10 +114,6 @@ func (sb *SectorBuilder) GetAllStagedSectors() ([]uint64, error) { return out, nil } -func (sb *SectorBuilder) GetAllSealedSectors() ([]SealedSectorMetadata, error) { - return sectorbuilder.GetAllSealedSectors(sb.handle) -} - func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]byte, error) { // Wait, this is a blocking method with no way of interrupting it? // does it checkpoint itself? diff --git a/storage/miner.go b/storage/miner.go index 4d16315f4..e563edaa4 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -4,7 +4,6 @@ import ( "context" "sync" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" @@ -118,15 +117,7 @@ func (m *Miner) commitUntrackedSectors(ctx context.Context) error { log.Warnf("Missing commitment for sector %d, committing sector", s.SectorID) - if err := m.commitSector(ctx, sectorbuilder.SectorSealingStatus{ - SectorID: s.SectorID, - State: sealing_state.Sealed, - CommD: s.CommD, - CommR: s.CommR, - Proof: s.Proof, - Pieces: s.Pieces, - Ticket: s.Ticket, - }); err != nil { + if err := m.commitSector(ctx, s); err != nil { log.Error("Committing uncommitted sector failed: ", err) } } diff --git a/storage/sector/store.go b/storage/sector/store.go index 5692c8649..30363ee67 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -293,8 +293,26 @@ func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.Sect return s.sb.SealStatus(sector) } -func (s *Store) Sealed() ([]sectorbuilder.SealedSectorMetadata, error) { - return s.sb.GetAllSealedSectors() +func (s *Store) Sealed() ([]sectorbuilder.SectorSealingStatus, error) { + l, err := s.sb.GetAllStagedSectors() + if err != nil { + return nil, err + } + + out := make([]sectorbuilder.SectorSealingStatus, 0) + for _, sid := range l { + status, err := s.sb.SealStatus(sid) + if err != nil { + return nil, err + } + + if status.State != sealing_state.Sealed { + continue + } + out = append(out, status) + } + + return out, nil } func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte, faults []uint64) ([]byte, error) { From 03ca08d9bfab9d707c9eef86652690a4534f6e3d Mon Sep 17 00:00:00 2001 From: wanghui Date: Wed, 30 Oct 2019 18:23:13 +0800 Subject: [PATCH 078/230] fix panic when close miner --- lib/jsonrpc/client.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 360e36aef..cb144de77 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -137,6 +137,7 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) // unpack chan type to make sure it's reflect.BothDir ctyp := reflect.ChanOf(reflect.BothDir, ftyp.Out(valOut).Elem()) ch := reflect.MakeChan(ctyp, 0) // todo: buffer? + chCtx, chCancel := context.WithCancel(ctx) retVal = ch.Convert(ftyp.Out(valOut)) buf := (&list.List{}).Init() @@ -144,6 +145,7 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) return ctx, func(result []byte, ok bool) { if !ok { + chCancel() // remote channel closed, close ours too ch.Close() return @@ -172,14 +174,18 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) go func() { for buf.Len() > 0 { - front := buf.Front() + select { + case <- chCtx.Done(): + buf.Init() + default: + front := buf.Front() + bufLk.Unlock() - bufLk.Unlock() + ch.Send(front.Value.(reflect.Value).Elem()) - ch.Send(front.Value.(reflect.Value).Elem()) // todo: select on ctx is probably a good idea - - bufLk.Lock() - buf.Remove(front) + bufLk.Lock() + buf.Remove(front) + } } bufLk.Unlock() From 3e5654d575ecc24a3584830137b0fa8b3b59e6ca Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 24 Oct 2019 21:39:13 +0800 Subject: [PATCH 079/230] initial actor changes for interactive porep --- build/params.go | 3 + chain/actors/actor_miner.go | 93 ++++++++++++---- chain/actors/cbor_gen.go | 215 ++++++++++++++++++++++++++++-------- gen/main.go | 1 + storage/miner.go | 7 +- 5 files changed, 250 insertions(+), 69 deletions(-) diff --git a/build/params.go b/build/params.go index e810ff45f..f69fb05ef 100644 --- a/build/params.go +++ b/build/params.go @@ -81,6 +81,9 @@ const PowerCollateralProportion = 5 const PerCapitaCollateralProportion = 1 const CollateralPrecision = 1000 +// Blocks +const InteractivePoRepDelay = 10 + // ///// // Devnet settings diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index baa35d0d5..9e5689d1b 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -2,6 +2,7 @@ package actors import ( "context" + "encoding/binary" "fmt" "github.com/filecoin-project/lotus/build" @@ -52,9 +53,9 @@ type StorageMinerActorState struct { // removal penalization is needed. NextDoneSet types.BitField - // Deals this miner has been slashed for since the last post submission. - //TODO: unsupported map key type "Cid" (if you want to use struct keys, your atlas needs a transform to string) - //ArbitratedDeals map[cid.Cid]struct{} + // UnprovenSectors is the set of sectors that have been committed to but not + // yet had their proofs submitted + UnprovenSectors map[string]*UnprovenSector // Amount of power this miner has. Power types.BigInt @@ -90,6 +91,12 @@ type MinerInfo struct { SectorSize uint64 } +type UnprovenSector struct { + CommD []byte + CommR []byte + SubmitHeight uint64 +} + type StorageMinerConstructorParams struct { Owner address.Address Worker address.Address @@ -205,12 +212,11 @@ type OnChainSealVerifyInfo struct { Epoch uint64 Proof []byte - DealIDs []uint64 SectorNumber uint64 } func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *OnChainSealVerifyInfo) ([]byte, ActorError) { - ctx := context.TODO() + ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { return nil, err @@ -225,20 +231,6 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, aerrors.New(1, "not authorized to commit sector for miner") } - // TODO: this needs to get normalized to either the ID address or the actor address - maddr := vmctx.Message().To - - ticket, err := vmctx.GetRandomness(params.Epoch) - if err != nil { - return nil, aerrors.Wrap(err, "failed to get randomness for commitsector") - } - - if ok, err := ValidatePoRep(maddr, mi.SectorSize, params, ticket); err != nil { - return nil, err - } else if !ok { - return nil, aerrors.New(2, "bad proof!") - } - // make sure the miner isnt trying to submit a pre-existing sector unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber) if err != nil { @@ -257,10 +249,67 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, aerrors.New(4, "not enough collateral") } + self.UnprovenSectors[uintToStringKey(params.SectorNumber)] = &UnprovenSector{ + CommR: params.CommR, + CommD: params.CommD, + SubmitHeight: vmctx.BlockHeight(), + } + + nstate, err := vmctx.Storage().Put(self) + if err != nil { + return nil, err + } + if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { + return nil, err + } + + return nil, nil +} + +func uintToStringKey(i uint64) string { + buf := make([]byte, 10) + n := binary.PutUvarint(buf, i) + return string(buf[:n]) +} + +type SubmitSectorProofParams struct { + Proof []byte + SectorID uint64 + DealIDs []uint64 +} + +func (sma StorageMinerActor) SubmitSectorProof(act *types.Actor, vmctx types.VMContext, params *SubmitSectorProofParams) ([]byte, ActorError) { + ctx := vmctx.Context() + oldstate, self, err := loadState(vmctx) + if err != nil { + return nil, err + } + + mi, err := loadMinerInfo(vmctx, self) + if err != nil { + return nil, err + } + + us, ok := self.UnprovenSectors[uintToStringKey(params.SectorID)] + if !ok { + return nil, aerrors.New(1, "no pre-commitment found for sector") + } + delete(self.UnprovenSectors, uintToStringKey(params.SectorID)) + + // TODO: ensure normalization to ID address + maddr := vmctx.Message().To + + rand, err := vmctx.GetRandomness(us.SubmitHeight + build.InteractivePoRepDelay) + if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, rand, params.Proof, params.SectorID); err != nil { + return nil, err + } else if !ok { + return nil, aerrors.New(2, "bad proof!") + } + // Note: There must exist a unique index in the miner's sector set for each // sector ID. The `faults`, `recovered`, and `done` parameters of the // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber, params.CommR, params.CommD) + nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.CommR, us.CommD) if err != nil { return nil, err } @@ -525,8 +574,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo, ticket []byte) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, maddr, ticket, params.SectorNumber, params.Proof) +func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof []byte, sectorID uint64) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, sectorID, proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 5444dafb0..3a87b61d5 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -197,7 +197,7 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{140}); err != nil { + if _, err := w.Write([]byte{141}); err != nil { return err } @@ -244,6 +244,26 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return err } + // t.t.UnprovenSectors (map[string]*actors.UnprovenSector) + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.UnprovenSectors))); err != nil { + return err + } + + for k, v := range t.UnprovenSectors { + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + + } + // t.t.Power (types.BigInt) if err := t.Power.MarshalCBOR(w); err != nil { return err @@ -277,7 +297,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 13 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -361,6 +381,59 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return err } + } + // t.t.UnprovenSectors (map[string]*actors.UnprovenSector) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("expected a map (major type 5)") + } + if extra > 4096 { + return fmt.Errorf("t.UnprovenSectors: map too large") + } + + t.UnprovenSectors = make(map[string]*UnprovenSector, extra) + + for i, l := 0, int(extra); i < l; i++ { + + var k string + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + k = string(sval) + } + + var v *UnprovenSector + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + v = new(UnprovenSector) + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + + t.UnprovenSectors[k] = v + } // t.t.Power (types.BigInt) @@ -497,7 +570,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{133}); err != nil { return err } @@ -530,16 +603,6 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { - return err - } - for _, v := range t.DealIDs { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { - return err - } - } - // t.t.SectorNumber (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorNumber)); err != nil { return err @@ -558,7 +621,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 5 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -623,36 +686,6 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Proof); err != nil { return err } - // t.t.DealIDs ([]uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.DealIDs: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.DealIDs = make([]uint64, extra) - } - for i := 0; i < int(extra); i++ { - - maj, val, err := cbg.CborReadHeader(br) - if err != nil { - return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) - } - - if maj != cbg.MajUnsignedInt { - return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) - } - - t.DealIDs[i] = val - } - // t.t.SectorNumber (uint64) maj, extra, err = cbg.CborReadHeader(br) @@ -666,6 +699,100 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *UnprovenSector) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.CommD ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { + return err + } + if _, err := w.Write(t.CommD); err != nil { + return err + } + + // t.t.CommR ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { + return err + } + if _, err := w.Write(t.CommR); err != nil { + return err + } + + // t.t.SubmitHeight (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SubmitHeight)); err != nil { + return err + } + return nil +} + +func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.CommD ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommD: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommD = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommD); err != nil { + return err + } + // t.t.CommR ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommR: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommR = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommR); err != nil { + return err + } + // t.t.SubmitHeight (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SubmitHeight = extra + return nil +} + func (t *MinerInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) diff --git a/gen/main.go b/gen/main.go index 127336a0b..fa8603a33 100644 --- a/gen/main.go +++ b/gen/main.go @@ -51,6 +51,7 @@ func main() { actors.StorageMinerActorState{}, actors.StorageMinerConstructorParams{}, actors.OnChainSealVerifyInfo{}, + actors.UnprovenSector{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, actors.PaymentVerifyParams{}, diff --git a/storage/miner.go b/storage/miner.go index e563edaa4..6d2a6776a 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -170,18 +170,19 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal log.Error("seal we just created failed verification") } - deals, err := m.secst.DealsForCommit(sinfo.SectorID) + // TODO: 2 stage commit + /*deals, err := m.secst.DealsForCommit(sinfo.SectorID) if err != nil { return xerrors.Errorf("getting sector deals failed: %w", err) } - + */ params := &actors.OnChainSealVerifyInfo{ CommD: sinfo.CommD[:], CommR: sinfo.CommR[:], Proof: sinfo.Proof, Epoch: sinfo.Ticket.BlockHeight, - DealIDs: deals, + //DealIDs: deals, SectorNumber: sinfo.SectorID, } enc, aerr := actors.SerializeParams(params) From a4c0610cc3013a00a7301f21030e79a4490c56ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Oct 2019 16:55:55 +0100 Subject: [PATCH 080/230] actors: Rename some storageminer stuff to match the spec --- chain/actors/actor_miner.go | 89 ++++++------ chain/actors/actor_storagepower.go | 1 + chain/actors/cbor_gen.go | 212 +++++++++++++---------------- gen/main.go | 2 +- storage/miner.go | 4 +- 5 files changed, 143 insertions(+), 165 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 9e5689d1b..28cc979a1 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -22,22 +22,27 @@ import ( type StorageMinerActor struct{} type StorageMinerActorState struct { - // Contains mostly static info about this miner - Info cid.Cid - - // Collateral that is waiting to be withdrawn. - DePledgedCollateral types.BigInt - - // Time at which the depledged collateral may be withdrawn. - DePledgeTime types.BigInt + // PreCommittedSectors is the set of sectors that have been committed to but not + // yet had their proofs submitted + PreCommittedSectors map[string]*UnprovenSector // All sectors this miner has committed. Sectors cid.Cid + // TODO: Spec says 'StagedCommittedSectors', which one is it? + // Sectors this miner is currently mining. It is only updated // when a PoSt is submitted (not as each new sector commitment is added). ProvingSet cid.Cid + // TODO: these: + // SectorTable + // SectorExpirationQueue + // ChallengeStatus + + // Contains mostly static info about this miner + Info cid.Cid + // Faulty sectors reported since last SubmitPost, // up to the current proving period's challenge time. CurrentFaultSet types.BitField @@ -53,10 +58,6 @@ type StorageMinerActorState struct { // removal penalization is needed. NextDoneSet types.BitField - // UnprovenSectors is the set of sectors that have been committed to but not - // yet had their proofs submitted - UnprovenSectors map[string]*UnprovenSector - // Amount of power this miner has. Power types.BigInt @@ -89,6 +90,8 @@ type MinerInfo struct { // Amount of space in each sector committed to the network by this miner. SectorSize uint64 + + // SubsectorCount } type UnprovenSector struct { @@ -106,7 +109,8 @@ type StorageMinerConstructorParams struct { type maMethods struct { Constructor uint64 - CommitSector uint64 + PreCommitSector uint64 + ProveCommitSector uint64 SubmitPoSt uint64 SlashStorageFault uint64 GetCurrentProvingSet uint64 @@ -121,32 +125,33 @@ type maMethods struct { ChangeWorker uint64 IsSlashed uint64 IsLate uint64 - AddFaults uint64 + DeclareFaults uint64 SlashConsensusFault uint64 } -var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18} +var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19} func (sma StorageMinerActor) Exports() []interface{} { return []interface{}{ 1: sma.StorageMinerConstructor, - 2: sma.CommitSector, - 3: sma.SubmitPoSt, - //4: sma.SlashStorageFault, - //5: sma.GetCurrentProvingSet, - //6: sma.ArbitrateDeal, - //7: sma.DePledge, - 8: sma.GetOwner, - 9: sma.GetWorkerAddr, - 10: sma.GetPower, - 11: sma.GetPeerID, - 12: sma.GetSectorSize, - 13: sma.UpdatePeerID, - //14: sma.ChangeWorker, - //15: sma.IsSlashed, - //16: sma.IsLate, - 17: sma.AddFaults, - 18: sma.SlashConsensusFault, + 2: sma.PreCommitSector, + 3: sma.ProveCommitSector, + 4: sma.SubmitPoSt, + //5: sma.SlashStorageFault, + //6: sma.GetCurrentProvingSet, + //7: sma.ArbitrateDeal, + //8: sma.DePledge, + 9: sma.GetOwner, + 10: sma.GetWorkerAddr, + 11: sma.GetPower, // TODO: Remove + 12: sma.GetPeerID, + 13: sma.GetSectorSize, + 14: sma.UpdatePeerID, + //15: sma.ChangeWorker, + //16: sma.IsSlashed, + //17: sma.IsLate, + 18: sma.DeclareFaults, + 19: sma.SlashConsensusFault, } } @@ -206,7 +211,7 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ return nil, nil } -type OnChainSealVerifyInfo struct { +type SectorPreCommitInfo struct { CommD []byte // TODO: update proofs code CommR []byte @@ -215,7 +220,7 @@ type OnChainSealVerifyInfo struct { SectorNumber uint64 } -func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *OnChainSealVerifyInfo) ([]byte, ActorError) { +func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorPreCommitInfo) ([]byte, ActorError) { ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { @@ -249,7 +254,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex return nil, aerrors.New(4, "not enough collateral") } - self.UnprovenSectors[uintToStringKey(params.SectorNumber)] = &UnprovenSector{ + self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &UnprovenSector{ CommR: params.CommR, CommD: params.CommD, SubmitHeight: vmctx.BlockHeight(), @@ -272,13 +277,13 @@ func uintToStringKey(i uint64) string { return string(buf[:n]) } -type SubmitSectorProofParams struct { +type SectorProveCommitInfo struct { Proof []byte SectorID uint64 DealIDs []uint64 } -func (sma StorageMinerActor) SubmitSectorProof(act *types.Actor, vmctx types.VMContext, params *SubmitSectorProofParams) ([]byte, ActorError) { +func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) { ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) if err != nil { @@ -290,11 +295,11 @@ func (sma StorageMinerActor) SubmitSectorProof(act *types.Actor, vmctx types.VMC return nil, err } - us, ok := self.UnprovenSectors[uintToStringKey(params.SectorID)] + us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)] if !ok { return nil, aerrors.New(1, "no pre-commitment found for sector") } - delete(self.UnprovenSectors, uintToStringKey(params.SectorID)) + delete(self.PreCommittedSectors, uintToStringKey(params.SectorID)) // TODO: ensure normalization to ID address maddr := vmctx.Message().To @@ -319,7 +324,7 @@ func (sma StorageMinerActor) SubmitSectorProof(act *types.Actor, vmctx types.VMC // Note: As written here, every miners first PoSt will only be over one sector. // We could set up a 'grace period' for starting mining that would allow miners // to submit several sectors for their first proving period. Alternatively, we - // could simply make the 'CommitSector' call take multiple sectors at a time. + // could simply make the 'PreCommitSector' call take multiple sectors at a time. // // Note: Proving period is a function of sector size; small sectors take less // time to prove than large sectors do. Sector size is selected when pledging. @@ -697,11 +702,11 @@ type PaymentVerifyParams struct { Proof []byte } -type AddFaultsParams struct { +type DeclareFaultsParams struct { Faults types.BitField } -func (sma StorageMinerActor) AddFaults(act *types.Actor, vmctx types.VMContext, params *AddFaultsParams) ([]byte, ActorError) { +func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { oldstate, self, aerr := loadState(vmctx) if aerr != nil { return nil, aerr diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index ea564bc62..e3f7cbfd3 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -329,6 +329,7 @@ func powerLookup(ctx context.Context, vmctx types.VMContext, self *StoragePowerS return types.EmptyInt, aerrors.New(1, "miner not registered with storage power actor") } + // TODO: Use local amt ret, err := vmctx.Send(miner, MAMethods.GetPower, types.NewInt(0), nil) if err != nil { return types.EmptyInt, aerrors.Wrap(err, "invoke Miner.GetPower") diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 3a87b61d5..84de28117 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -197,24 +197,28 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{141}); err != nil { + if _, err := w.Write([]byte{139}); err != nil { return err } - // t.t.Info (cid.Cid) - - if err := cbg.WriteCid(w, t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) - } - - // t.t.DePledgedCollateral (types.BigInt) - if err := t.DePledgedCollateral.MarshalCBOR(w); err != nil { + // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { return err } - // t.t.DePledgeTime (types.BigInt) - if err := t.DePledgeTime.MarshalCBOR(w); err != nil { - return err + for k, v := range t.PreCommittedSectors { + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + } // t.t.Sectors (cid.Cid) @@ -229,6 +233,12 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.ProvingSet: %w", err) } + // t.t.Info (cid.Cid) + + if err := cbg.WriteCid(w, t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + // t.t.CurrentFaultSet (types.BitField) if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { return err @@ -244,26 +254,6 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.UnprovenSectors (map[string]*actors.UnprovenSector) - if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.UnprovenSectors))); err != nil { - return err - } - - for k, v := range t.UnprovenSectors { - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { - return err - } - if _, err := w.Write([]byte(k)); err != nil { - return err - } - - if err := v.MarshalCBOR(w); err != nil { - return err - } - - } - // t.t.Power (types.BigInt) if err := t.Power.MarshalCBOR(w); err != nil { return err @@ -297,92 +287,11 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 13 { + if extra != 11 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Info (cid.Cid) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) - } - - t.Info = c - - } - // t.t.DePledgedCollateral (types.BigInt) - - { - - if err := t.DePledgedCollateral.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.DePledgeTime (types.BigInt) - - { - - if err := t.DePledgeTime.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.Sectors (cid.Cid) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) - } - - t.Sectors = c - - } - // t.t.ProvingSet (cid.Cid) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ProvingSet: %w", err) - } - - t.ProvingSet = c - - } - // t.t.CurrentFaultSet (types.BitField) - - { - - if err := t.CurrentFaultSet.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.NextFaultSet (types.BitField) - - { - - if err := t.NextFaultSet.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.NextDoneSet (types.BitField) - - { - - if err := t.NextDoneSet.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.UnprovenSectors (map[string]*actors.UnprovenSector) + // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -392,10 +301,10 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("expected a map (major type 5)") } if extra > 4096 { - return fmt.Errorf("t.UnprovenSectors: map too large") + return fmt.Errorf("t.PreCommittedSectors: map too large") } - t.UnprovenSectors = make(map[string]*UnprovenSector, extra) + t.PreCommittedSectors = make(map[string]*UnprovenSector, extra) for i, l := 0, int(extra); i < l; i++ { @@ -432,7 +341,70 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } - t.UnprovenSectors[k] = v + t.PreCommittedSectors[k] = v + + } + // t.t.Sectors (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) + } + + t.Sectors = c + + } + // t.t.ProvingSet (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ProvingSet: %w", err) + } + + t.ProvingSet = c + + } + // t.t.Info (cid.Cid) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = c + + } + // t.t.CurrentFaultSet (types.BitField) + + { + + if err := t.CurrentFaultSet.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.NextFaultSet (types.BitField) + + { + + if err := t.NextFaultSet.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.NextDoneSet (types.BitField) + + { + + if err := t.NextDoneSet.UnmarshalCBOR(br); err != nil { + return err + } } // t.t.Power (types.BigInt) @@ -565,7 +537,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { +func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -610,7 +582,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { +func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) diff --git a/gen/main.go b/gen/main.go index fa8603a33..102872f08 100644 --- a/gen/main.go +++ b/gen/main.go @@ -50,7 +50,7 @@ func main() { actors.AccountActorState{}, actors.StorageMinerActorState{}, actors.StorageMinerConstructorParams{}, - actors.OnChainSealVerifyInfo{}, + actors.SectorPreCommitInfo{}, actors.UnprovenSector{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, diff --git a/storage/miner.go b/storage/miner.go index 6d2a6776a..540d662c7 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -176,7 +176,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal return xerrors.Errorf("getting sector deals failed: %w", err) } */ - params := &actors.OnChainSealVerifyInfo{ + params := &actors.SectorPreCommitInfo{ CommD: sinfo.CommD[:], CommR: sinfo.CommR[:], Proof: sinfo.Proof, @@ -193,7 +193,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal msg := &types.Message{ To: m.maddr, From: m.worker, - Method: actors.MAMethods.CommitSector, + Method: actors.MAMethods.PreCommitSector, Params: enc, Value: types.NewInt(0), // TODO: need to ensure sufficient collateral GasLimit: types.NewInt(1000000 /* i dont know help */), From 3ea0997c93b812c29a564055c48fccc2cf70afd1 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 30 Oct 2019 17:38:39 +0100 Subject: [PATCH 081/230] Add support for different configs License: MIT Signed-off-by: Jakub Sztandera --- chain/gen/gen.go | 2 +- cmd/lotus-storage-miner/init.go | 4 +- cmd/lotus/daemon.go | 5 +- node/builder.go | 82 ++++++++++++++++++++++++--------- node/config/def.go | 49 ++++++++++++++++---- node/config/load.go | 27 ++++++++--- node/config/load_test.go | 14 +++--- node/node_test.go | 2 +- node/repo/fsrepo.go | 71 ++++++++++++++++++++++------ node/repo/fsrepo_test.go | 2 +- node/repo/interface.go | 5 +- node/repo/memrepo.go | 15 +++--- node/repo/repo_test.go | 10 ++-- 13 files changed, 207 insertions(+), 81 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index b38f2c0df..e1452bb91 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -75,7 +75,7 @@ func (m mybs) Get(c cid.Cid) (block.Block, error) { func NewGenerator() (*ChainGen, error) { mr := repo.NewMemory(nil) - lr, err := mr.Lock() + lr, err := mr.Lock(repo.RepoStorageMiner) if err != nil { return nil, xerrors.Errorf("taking mem-repo lock failed: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 54d488a1b..5943d6dac 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -101,7 +101,7 @@ var initCmd = &cli.Command{ log.Info("Initializing repo") - if err := r.Init(); err != nil { + if err := r.Init(repo.RepoStorageMiner); err != nil { return err } @@ -122,7 +122,7 @@ var initCmd = &cli.Command{ } func storageMinerInit(ctx context.Context, cctx *cli.Context, api api.FullNode, r repo.Repo) error { - lr, err := r.Lock() + lr, err := r.Lock(repo.RepoStorageMiner) if err != nil { return err } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c9b736c72..4921f3930 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -4,9 +4,10 @@ package main import ( "context" - "github.com/filecoin-project/lotus/peermgr" "io/ioutil" + "github.com/filecoin-project/lotus/peermgr" + "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -53,7 +54,7 @@ var DaemonCmd = &cli.Command{ return err } - if err := r.Init(); err != nil && err != repo.ErrRepoExists { + if err := r.Init(repo.RepoFullNode); err != nil && err != repo.ErrRepoExists { return err } diff --git a/node/builder.go b/node/builder.go index c316139ee..f704bef19 100644 --- a/node/builder.go +++ b/node/builder.go @@ -15,6 +15,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" "go.uber.org/fx" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" @@ -100,11 +101,6 @@ const ( _nInvokes // keep this last ) -const ( - nodeFull = iota - nodeStorageMiner -) - type Settings struct { // modules is a map of constructors for DI // @@ -117,14 +113,12 @@ type Settings struct { // type, and must be applied in correct order invokes []fx.Option - nodeType int + nodeType repo.RepoType Online bool // Online option applied Config bool // Config option applied } -var defConf = config.Default() - func defaults() []Option { return []Option{ Override(new(helpers.MetricsCtx), context.Background), @@ -161,10 +155,14 @@ func libp2p() Option { Override(new(*pubsub.PubSub), lp2p.GossipSub()), Override(PstoreAddSelfKeysKey, lp2p.PstoreAddSelfKeys), - Override(StartListeningKey, lp2p.StartListening(defConf.Libp2p.ListenAddresses)), + Override(StartListeningKey, lp2p.StartListening(config.DefaultFullNode().Libp2p.ListenAddresses)), ) } +func isType(t repo.RepoType) func(s *Settings) bool { + return func(s *Settings) bool { return s.nodeType == t } +} + // Online sets up basic libp2p node func Online() Option { return Options( @@ -181,7 +179,7 @@ func Online() Option { // Full node - ApplyIf(func(s *Settings) bool { return s.nodeType == nodeFull }, + ApplyIf(isType(repo.RepoFullNode), // TODO: Fix offline mode Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap), @@ -230,7 +228,7 @@ func Online() Option { ), // Storage miner - ApplyIf(func(s *Settings) bool { return s.nodeType == nodeStorageMiner }, + ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), Override(new(*sector.Store), sector.NewStore), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), @@ -260,7 +258,7 @@ func StorageMiner(out *api.StorageMiner) Option { ), func(s *Settings) error { - s.nodeType = nodeStorageMiner + s.nodeType = repo.RepoStorageMiner return nil }, @@ -274,7 +272,7 @@ func StorageMiner(out *api.StorageMiner) Option { } // Config sets up constructors based on the provided Config -func Config(cfg *config.Root) Option { +func ConfigCommon(cfg *config.Common) Option { return Options( func(s *Settings) error { s.Config = true; return nil }, @@ -284,27 +282,64 @@ func Config(cfg *config.Root) Option { ApplyIf(func(s *Settings) bool { return len(cfg.Libp2p.BootstrapPeers) > 0 }, Override(new(dtypes.BootstrapPeers), modules.ConfigBootstrap(cfg.Libp2p.BootstrapPeers)), ), - - ApplyIf(func(s *Settings) bool { return s.nodeType == nodeFull }, - Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), - ), ), ) } -func Repo(r repo.Repo) Option { - lr, err := r.Lock() +func ConfigFullNode(cfg *config.FullNode) Option { + //ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoFullNode }), + return Options( + ConfigCommon(&cfg.Common), + Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)), + ) +} + +func repoFull(r repo.Repo) Option { + lr, err := r.Lock(repo.RepoFullNode) if err != nil { return Error(err) } - cfg, err := lr.Config() + c, err := lr.Config() + if err != nil { + return Error(err) + } + cfg, ok := c.(*config.FullNode) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + + return Options( + ConfigFullNode(cfg), + Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing + ) +} + +func repoMiner(r repo.Repo) Option { + lr, err := r.Lock(repo.RepoStorageMiner) + if err != nil { + return Error(err) + } + c, err := lr.Config() if err != nil { return Error(err) } + cfg, ok := c.(*config.StorageMiner) + if !ok { + return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) + } + return Options( - Config(cfg), + ConfigCommon(&cfg.Common), Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing + ) +} + +func Repo(r repo.Repo) Option { + + return Options( + ApplyIf(isType(repo.RepoFullNode), repoFull(r)), + ApplyIf(isType(repo.RepoStorageMiner), repoMiner(r)), Override(new(dtypes.MetadataDS), modules.Datastore), Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), @@ -337,8 +372,9 @@ type StopFunc func(context.Context) error // New builds and starts new Filecoin node func New(ctx context.Context, opts ...Option) (StopFunc, error) { settings := Settings{ - modules: map[interface{}]fx.Option{}, - invokes: make([]fx.Option, _nInvokes), + modules: map[interface{}]fx.Option{}, + invokes: make([]fx.Option, _nInvokes), + nodeType: repo.RepoFullNode, } // apply module options in the right order diff --git a/node/config/def.go b/node/config/def.go index 3b1020840..93a876656 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -1,15 +1,27 @@ package config -import "time" +import ( + "encoding" + "time" +) -// Root is starting point of the config -type Root struct { +// Common is common config between full node and miner +type Common struct { API API Libp2p Libp2p +} +// FullNode is a full node config +type FullNode struct { + Common Metrics Metrics } +// StorageMiner is a storage miner config +type StorageMiner struct { + Common +} + // API contains configs for API endpoint type API struct { ListenAddress string @@ -26,9 +38,8 @@ type Metrics struct { Nickname string } -// Default returns the default config -func Default() *Root { - def := Root{ +func defCommon() Common { + return Common{ API: API{ ListenAddress: "/ip6/::1/tcp/1234/http", Timeout: Duration(30 * time.Second), @@ -40,10 +51,27 @@ func Default() *Root { }, }, } - return &def + } -// Duration is a wrapper type for time.Duration for decoding it from TOML +// Default returns the default config +func DefaultFullNode() *FullNode { + return &FullNode{ + Common: defCommon(), + } +} + +func DefaultStorageMiner() *StorageMiner { + return &StorageMiner{ + Common: defCommon(), + } +} + +var _ encoding.TextMarshaler = (*Duration)(nil) +var _ encoding.TextUnmarshaler = (*Duration)(nil) + +// Duration is a wrapper type for time.Duration +// for decoding and encoding from/to TOML type Duration time.Duration // UnmarshalText implements interface for TOML decoding @@ -55,3 +83,8 @@ func (dur *Duration) UnmarshalText(text []byte) error { *dur = Duration(d) return err } + +func (dur Duration) MarshalText() ([]byte, error) { + d := time.Duration(dur) + return []byte(d.String()), nil +} diff --git a/node/config/load.go b/node/config/load.go index 2f2d223b3..fd1aeb181 100644 --- a/node/config/load.go +++ b/node/config/load.go @@ -1,30 +1,32 @@ package config import ( + "bytes" "io" "os" "github.com/BurntSushi/toml" + "golang.org/x/xerrors" ) // FromFile loads config from a specified file overriding defaults specified in -// the source code. If file does not exist or is empty defaults are asummed. -func FromFile(path string) (*Root, error) { +// the def parameter. If file does not exist or is empty defaults are asummed. +func FromFile(path string, def interface{}) (interface{}, error) { file, err := os.Open(path) switch { case os.IsNotExist(err): - return Default(), nil + return def, nil case err != nil: return nil, err } defer file.Close() //nolint:errcheck // The file is RO - return FromReader(file) + return FromReader(file, def) } // FromReader loads config from a reader instance. -func FromReader(reader io.Reader) (*Root, error) { - cfg := Default() +func FromReader(reader io.Reader, def interface{}) (interface{}, error) { + cfg := def _, err := toml.DecodeReader(reader, cfg) if err != nil { return nil, err @@ -32,3 +34,16 @@ func FromReader(reader io.Reader) (*Root, error) { return cfg, nil } + +func ConfigComment(t interface{}) ([]byte, error) { + buf := new(bytes.Buffer) + _, _ = buf.WriteString("# Default config:\n") + e := toml.NewEncoder(buf) + if err := e.Encode(t); err != nil { + return nil, xerrors.Errorf("encoding config: %w", err) + } + b := buf.Bytes() + b = bytes.ReplaceAll(b, []byte("\n"), []byte("\n#")) + return b, nil + +} diff --git a/node/config/load_test.go b/node/config/load_test.go index 543a00103..9abe8a54b 100644 --- a/node/config/load_test.go +++ b/node/config/load_test.go @@ -14,16 +14,16 @@ func TestDecodeNothing(t *testing.T) { assert := assert.New(t) { - cfg, err := FromFile(os.DevNull) + cfg, err := FromFile(os.DevNull, DefaultFullNode()) assert.Nil(err, "error should be nil") - assert.Equal(Default(), cfg, + assert.Equal(DefaultFullNode(), cfg, "config from empty file should be the same as default") } { - cfg, err := FromFile("./does-not-exist.toml") + cfg, err := FromFile("./does-not-exist.toml", DefaultFullNode()) assert.Nil(err, "error should be nil") - assert.Equal(Default(), cfg, + assert.Equal(DefaultFullNode(), cfg, "config from not exisiting file should be the same as default") } } @@ -34,11 +34,11 @@ func TestParitalConfig(t *testing.T) { [API] Timeout = "10s" ` - expected := Default() + expected := DefaultFullNode() expected.API.Timeout = Duration(10 * time.Second) { - cfg, err := FromReader(bytes.NewReader([]byte(cfgString))) + cfg, err := FromReader(bytes.NewReader([]byte(cfgString)), DefaultFullNode()) assert.NoError(err, "error should be nil") assert.Equal(expected, cfg, "config from reader should contain changes") @@ -55,7 +55,7 @@ func TestParitalConfig(t *testing.T) { assert.NoError(err, "closing tmp file should not error") defer os.Remove(fname) //nolint:errcheck - cfg, err := FromFile(fname) + cfg, err := FromFile(fname, DefaultFullNode()) assert.Nil(err, "error should be nil") assert.Equal(expected, cfg, "config from reader should contain changes") diff --git a/node/node_test.go b/node/node_test.go index 06f1fc1c6..82ab996bd 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -34,7 +34,7 @@ import ( func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode { r := repo.NewMemory(nil) - lr, err := r.Lock() + lr, err := r.Lock(repo.RepoStorageMiner) require.NoError(t, err) pk, _, err := crypto.GenerateEd25519Key(rand.Reader) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index df496315c..0a81e6513 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -2,6 +2,7 @@ package repo import ( "encoding/json" + "fmt" "io" "io/ioutil" "os" @@ -33,13 +34,33 @@ const ( fsKeystore = "keystore" ) +type RepoType int + +const ( + _ = iota // Default is invalid + RepoFullNode RepoType = iota + RepoStorageMiner +) + +func defConfForType(t RepoType) interface{} { + switch t { + case RepoFullNode: + return config.DefaultFullNode() + case RepoStorageMiner: + return config.DefaultStorageMiner() + default: + panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) + } +} + var log = logging.Logger("repo") var ErrRepoExists = xerrors.New("repo exists") // FsRepo is struct for repo, use NewFS to create type FsRepo struct { - path string + path string + repoType RepoType } var _ Repo = &FsRepo{} @@ -65,7 +86,7 @@ func (fsr *FsRepo) Exists() (bool, error) { return !notexist, err } -func (fsr *FsRepo) Init() error { +func (fsr *FsRepo) Init(t RepoType) error { exist, err := fsr.Exists() if err != nil { return err @@ -79,18 +100,36 @@ func (fsr *FsRepo) Init() error { if err != nil && !os.IsExist(err) { return err } - c, err := os.Create(filepath.Join(fsr.path, fsConfig)) - if err != nil { - return err - } - if err := c.Close(); err != nil { - return err + + if err := fsr.initConfig(t); err != nil { + return xerrors.Errorf("init config: %w", err) } return fsr.initKeystore() } +func (fsr *FsRepo) initConfig(t RepoType) error { + c, err := os.Create(filepath.Join(fsr.path, fsConfig)) + if err != nil { + return err + } + + comm, err := config.ConfigComment(defConfForType(t)) + if err != nil { + return xerrors.Errorf("comment: %w", err) + } + _, err = c.Write(comm) + if err != nil { + return xerrors.Errorf("write config: %w", err) + } + + if err := c.Close(); err != nil { + return xerrors.Errorf("close config: %w", err) + } + return nil +} + func (fsr *FsRepo) initKeystore() error { kstorePath := filepath.Join(fsr.path, fsKeystore) if _, err := os.Stat(kstorePath); err == nil { @@ -142,7 +181,7 @@ func (fsr *FsRepo) APIToken() ([]byte, error) { } // Lock acquires exclusive lock on this repo -func (fsr *FsRepo) Lock() (LockedRepo, error) { +func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) { locked, err := fslock.Locked(fsr.path, fsLock) if err != nil { return nil, xerrors.Errorf("could not check lock status: %w", err) @@ -156,14 +195,16 @@ func (fsr *FsRepo) Lock() (LockedRepo, error) { return nil, xerrors.Errorf("could not lock the repo: %w", err) } return &fsLockedRepo{ - path: fsr.path, - closer: closer, + path: fsr.path, + repoType: repoType, + closer: closer, }, nil } type fsLockedRepo struct { - path string - closer io.Closer + path string + repoType RepoType + closer io.Closer ds datastore.Batching dsErr error @@ -219,11 +260,11 @@ func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(fsr.ds, datastore.NewKey(ns)), nil } -func (fsr *fsLockedRepo) Config() (*config.Root, error) { +func (fsr *fsLockedRepo) Config() (interface{}, error) { if err := fsr.stillValid(); err != nil { return nil, err } - return config.FromFile(fsr.join(fsConfig)) + return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType)) } func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go index 938314a23..de55eb351 100644 --- a/node/repo/fsrepo_test.go +++ b/node/repo/fsrepo_test.go @@ -17,7 +17,7 @@ func genFsRepo(t *testing.T) (*FsRepo, func()) { t.Fatal(err) } - err = repo.Init() + err = repo.Init(RepoFullNode) if err != ErrRepoExists && err != nil { t.Fatal(err) } diff --git a/node/repo/interface.go b/node/repo/interface.go index 9fabc32d0..9a7a3e3eb 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -7,7 +7,6 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/config" ) var ( @@ -25,7 +24,7 @@ type Repo interface { APIToken() ([]byte, error) // Lock locks the repo for exclusive use. - Lock() (LockedRepo, error) + Lock(RepoType) (LockedRepo, error) } type LockedRepo interface { @@ -36,7 +35,7 @@ type LockedRepo interface { Datastore(namespace string) (datastore.Batching, error) // Returns config in this repo - Config() (*config.Root, error) + Config() (interface{}, error) // SetAPIEndpoint sets the endpoint of the current API // so it can be read by API clients diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 84c52ed4f..00f6de424 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -10,7 +10,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/config" ) type MemRepo struct { @@ -24,12 +23,13 @@ type MemRepo struct { token *byte datastore datastore.Datastore - configF func() *config.Root + configF func(t RepoType) interface{} keystore map[string]types.KeyInfo } type lockedMemRepo struct { mem *MemRepo + t RepoType sync.RWMutex token *byte @@ -44,7 +44,7 @@ var _ Repo = &MemRepo{} // MemRepoOptions contains options for memory repo type MemRepoOptions struct { Ds datastore.Datastore - ConfigF func() *config.Root + ConfigF func(RepoType) interface{} KeyStore map[string]types.KeyInfo } @@ -56,7 +56,7 @@ func NewMemory(opts *MemRepoOptions) *MemRepo { opts = &MemRepoOptions{} } if opts.ConfigF == nil { - opts.ConfigF = config.Default + opts.ConfigF = defConfForType } if opts.Ds == nil { opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore()) @@ -92,7 +92,7 @@ func (mem *MemRepo) APIToken() ([]byte, error) { return mem.api.token, nil } -func (mem *MemRepo) Lock() (LockedRepo, error) { +func (mem *MemRepo) Lock(t RepoType) (LockedRepo, error) { select { case mem.repoLock <- struct{}{}: default: @@ -102,6 +102,7 @@ func (mem *MemRepo) Lock() (LockedRepo, error) { return &lockedMemRepo{ mem: mem, + t: t, token: mem.token, }, nil } @@ -143,11 +144,11 @@ func (lmem *lockedMemRepo) Datastore(ns string) (datastore.Batching, error) { return namespace.Wrap(lmem.mem.datastore, datastore.NewKey(ns)), nil } -func (lmem *lockedMemRepo) Config() (*config.Root, error) { +func (lmem *lockedMemRepo) Config() (interface{}, error) { if err := lmem.checkToken(); err != nil { return nil, err } - return lmem.mem.configF(), nil + return lmem.mem.configF(lmem.t), nil } func (lmem *lockedMemRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go index 34185a2c9..d9cdc5dc5 100644 --- a/node/repo/repo_test.go +++ b/node/repo/repo_test.go @@ -18,12 +18,12 @@ func basicTest(t *testing.T, repo Repo) { } assert.Nil(t, apima, "with no api endpoint, return should be nil") - lrepo, err := repo.Lock() + lrepo, err := repo.Lock(RepoFullNode) assert.NoError(t, err, "should be able to lock once") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") { - lrepo2, err := repo.Lock() + lrepo2, err := repo.Lock(RepoFullNode) if assert.Error(t, err) { assert.Equal(t, ErrRepoAlreadyLocked, err) } @@ -33,7 +33,7 @@ func basicTest(t *testing.T, repo Repo) { err = lrepo.Close() assert.NoError(t, err, "should be able to unlock") - lrepo, err = repo.Lock() + lrepo, err = repo.Lock(RepoFullNode) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") @@ -48,7 +48,7 @@ func basicTest(t *testing.T, repo Repo) { assert.Equal(t, ma, apima, "returned API multiaddr should be the same") cfg, err := lrepo.Config() - assert.Equal(t, config.Default(), cfg, "there should be a default config") + assert.Equal(t, config.DefaultFullNode(), cfg, "there should be a default config") assert.NoError(t, err, "config should not error") err = lrepo.Close() @@ -64,7 +64,7 @@ func basicTest(t *testing.T, repo Repo) { k1 := types.KeyInfo{Type: "foo"} k2 := types.KeyInfo{Type: "bar"} - lrepo, err = repo.Lock() + lrepo, err = repo.Lock(RepoFullNode) assert.NoError(t, err, "should be able to relock") assert.NotNil(t, lrepo, "locked repo shouldn't be nil") From 46d782b30be650f4600c99c1e62a83f0006b4c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Oct 2019 18:37:38 +0100 Subject: [PATCH 082/230] storageminer: Drop commitment tracker --- api/api.go | 10 -- api/struct.go | 6 - chain/deals/provider.go | 5 +- chain/deals/provider_states.go | 10 +- cmd/lotus-storage-miner/commitments.go | 41 ----- cmd/lotus-storage-miner/main.go | 1 - node/builder.go | 2 - node/impl/storminer.go | 6 - node/modules/storageminer.go | 5 +- storage/commitment/tracker.go | 198 ------------------------- storage/miner.go | 9 +- 11 files changed, 10 insertions(+), 283 deletions(-) delete mode 100644 cmd/lotus-storage-miner/commitments.go delete mode 100644 storage/commitment/tracker.go diff --git a/api/api.go b/api/api.go index 463e5e9cb..635901ecb 100644 --- a/api/api.go +++ b/api/api.go @@ -168,8 +168,6 @@ type StorageMiner interface { SectorsList(context.Context) ([]uint64, error) SectorsRefs(context.Context) (map[string][]SealedRef, error) - - CommitmentsList(context.Context) ([]SectorCommitment, error) } // Version provides various build-time information @@ -332,14 +330,6 @@ type SyncState struct { Height uint64 } -type SectorCommitment struct { - SectorID uint64 - Miner address.Address - - CommitMsg cid.Cid - DealIDs []uint64 -} - type SyncStateStage int const ( diff --git a/api/struct.go b/api/struct.go index 47d88aa69..6eefe41dc 100644 --- a/api/struct.go +++ b/api/struct.go @@ -133,8 +133,6 @@ type StorageMinerStruct struct { SectorsList func(context.Context) ([]uint64, error) `perm:"read"` SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - - CommitmentsList func(context.Context) ([]SectorCommitment, error) `perm:"read"` } } @@ -481,10 +479,6 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]Seal return c.Internal.SectorsRefs(ctx) } -func (c *StorageMinerStruct) CommitmentsList(ctx context.Context) ([]SectorCommitment, error) { - return c.Internal.CommitmentsList(ctx) -} - var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 86383c61b..356f7d2e7 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -43,7 +42,6 @@ type Provider struct { askLk sync.Mutex secst *sectorblocks.SectorBlocks - commt *commitment.Tracker full api.FullNode // TODO: Use a custom protocol or graphsync in the future @@ -70,7 +68,7 @@ type minerDealUpdate struct { mut func(*MinerDeal) } -func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *commitment.Tracker, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { +func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -82,7 +80,6 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt * h := &Provider{ secst: secst, - commt: commt, dag: dag, full: fullNode, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 595f236da..6c86eb8e9 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -290,12 +290,14 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { // TODO: Add dealID to commtracker (probably before sealing) - mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) + /*mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) if err != nil { log.Warnf("Waiting for sector commitment message: %s", err) - } + }*/ - err = p.sendSignedResponse(&Response{ + panic("fixme") + + /*err = p.sendSignedResponse(&Response{ State: api.DealComplete, Proposal: deal.ProposalCid, @@ -303,7 +305,7 @@ func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDea }) if err != nil { log.Warnf("Sending deal response failed: %s", err) - } + }*/ return nil, nil } diff --git a/cmd/lotus-storage-miner/commitments.go b/cmd/lotus-storage-miner/commitments.go deleted file mode 100644 index a7825deb1..000000000 --- a/cmd/lotus-storage-miner/commitments.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "fmt" - lcli "github.com/filecoin-project/lotus/cli" - - "gopkg.in/urfave/cli.v2" -) - -var commitmentsCmd = &cli.Command{ - Name: "commitments", - Usage: "interact with commitment tracker", - Subcommands: []*cli.Command{ - commitmentsListCmd, - }, -} - -var commitmentsListCmd = &cli.Command{ - Name: "list", - Usage: "List tracked sector commitments", - Action: func(cctx *cli.Context) error { - api, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - - ctx := lcli.ReqContext(cctx) - - comms, err := api.CommitmentsList(ctx) - if err != nil { - return err - } - - for _, comm := range comms { - fmt.Printf("%s:%d msg:%s, deals: %v\n", comm.Miner, comm.SectorID, comm.CommitMsg, comm.DealIDs) - } - - return nil - }, -} diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index b6aad1d73..f8bf2aac1 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -26,7 +26,6 @@ func main() { infoCmd, storeGarbageCmd, sectorsCmd, - commitmentsCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/node/builder.go b/node/builder.go index c316139ee..26b6bbd7f 100644 --- a/node/builder.go +++ b/node/builder.go @@ -40,7 +40,6 @@ import ( "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -234,7 +233,6 @@ func Online() Option { Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), Override(new(*sector.Store), sector.NewStore), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - Override(new(*commitment.Tracker), commitment.NewTracker), Override(new(sector.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 8eb363fe2..c763dab5e 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -24,7 +23,6 @@ type StorageMinerAPI struct { SectorBuilder *sectorbuilder.SectorBuilder Sectors *sector.Store SectorBlocks *sectorblocks.SectorBlocks - CommitmentTracker *commitment.Tracker Miner *storage.Miner } @@ -83,8 +81,4 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed return out, nil } -func (sm *StorageMinerAPI) CommitmentsList(ctx context.Context) ([]api.SectorCommitment, error) { - return sm.CommitmentTracker.List() -} - var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4f9df1c42..81b34522a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sector" ) @@ -72,13 +71,13 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, secst *sector.Store, commt *commitment.Tracker) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, secst *sector.Store) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, h, ds, secst, commt) + sm, err := storage.NewMiner(api, maddr, h, ds, secst) if err != nil { return nil, err } diff --git a/storage/commitment/tracker.go b/storage/commitment/tracker.go deleted file mode 100644 index 90e2459bf..000000000 --- a/storage/commitment/tracker.go +++ /dev/null @@ -1,198 +0,0 @@ -package commitment - -import ( - "context" - "fmt" - "strconv" - "strings" - "sync" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" - logging "github.com/ipfs/go-log" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/address" - "github.com/filecoin-project/lotus/node/modules/dtypes" - dsq "github.com/ipfs/go-datastore/query" -) - -var log = logging.Logger("commitment") - -func init() { - cbor.RegisterCborType(commitment{}) -} - -var commitmentDsPrefix = datastore.NewKey("/commitments") - -type Tracker struct { - commitments datastore.Datastore - - lk sync.Mutex - - waits map[datastore.Key]chan struct{} -} - -func NewTracker(ds dtypes.MetadataDS) *Tracker { - return &Tracker{ - commitments: namespace.Wrap(ds, commitmentDsPrefix), - waits: map[datastore.Key]chan struct{}{}, - } -} - -type commitment struct { - DealIDs []uint64 - Msg cid.Cid -} - -func commitmentKey(miner address.Address, sectorId uint64) datastore.Key { - return commitmentDsPrefix.ChildString(miner.String()).ChildString(fmt.Sprintf("%d", sectorId)) -} - -func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, commitMsg cid.Cid) error { - key := commitmentKey(miner, sectorId) - - ct.lk.Lock() - defer ct.lk.Unlock() - - tracking, err := ct.commitments.Get(key) - switch err { - case nil: - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return err - } - - if !comm.Msg.Equals(commitMsg) { - log.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg) - } - - log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg) - - // we still want to store it - fallthrough // TODO: ideally we'd keep around both (even though we'll - // usually only need the new one) - case datastore.ErrNotFound: - comm := &commitment{Msg: commitMsg} - commB, err := cbor.DumpObject(comm) - if err != nil { - return err - } - - if err := ct.commitments.Put(key, commB); err != nil { - return err - } - - waits, ok := ct.waits[key] - if ok { - close(waits) - delete(ct.waits, key) - } - return nil - default: - return err - } -} - -func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sectorId uint64) (cid.Cid, error) { - key := commitmentKey(miner, sectorId) - - ct.lk.Lock() - - tracking, err := ct.commitments.Get(key) - if err != datastore.ErrNotFound { - ct.lk.Unlock() - - if err != nil { - return cid.Undef, err - } - - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return cid.Undef, err - } - - return comm.Msg, nil - } - - wait, ok := ct.waits[key] - if !ok { - wait = make(chan struct{}) - ct.waits[key] = wait - } - - ct.lk.Unlock() - - select { - case <-wait: - tracking, err := ct.commitments.Get(key) - if err != nil { - return cid.Undef, xerrors.Errorf("failed to get commitment after waiting: %w", err) - } - - var comm commitment - if err := cbor.DecodeInto(tracking, &comm); err != nil { - return cid.Undef, err - } - - return comm.Msg, nil - case <-ctx.Done(): - return cid.Undef, ctx.Err() - } -} - -func (ct *Tracker) List() ([]api.SectorCommitment, error) { - out := make([]api.SectorCommitment, 0) - - ct.lk.Lock() - defer ct.lk.Unlock() - - res, err := ct.commitments.Query(dsq.Query{}) - if err != nil { - return nil, err - } - defer res.Close() - - for { - res, ok := res.NextSync() - if !ok { - break - } - - if res.Error != nil { - return nil, xerrors.Errorf("iterating commitments: %w", err) - } - - parts := strings.Split(res.Key, "/") - if len(parts) != 4 { - return nil, xerrors.Errorf("expected commitment key to be 4 parts, Key %s", res.Key) - } - - miner, err := address.NewFromString(parts[2]) - if err != nil { - return nil, xerrors.Errorf("parsing miner address: %w", err) - } - - sectorID, err := strconv.ParseInt(parts[3], 10, 64) - if err != nil { - return nil, xerrors.Errorf("parsing sector id: %w", err) - } - - var comm commitment - if err := cbor.DecodeInto(res.Value, &comm); err != nil { - return nil, xerrors.Errorf("decoding commitment %s (`% X`): %w", res.Key, res.Value, err) - } - - out = append(out, api.SectorCommitment{ - SectorID: uint64(sectorID), - Miner: miner, - CommitMsg: comm.Msg, - DealIDs: comm.DealIDs, - }) - } - - return out, nil -} diff --git a/storage/miner.go b/storage/miner.go index 540d662c7..089a46a3e 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/storage/commitment" "github.com/filecoin-project/lotus/storage/sector" ) @@ -31,7 +30,6 @@ type Miner struct { events *events.Events secst *sector.Store - commt *commitment.Tracker maddr address.Address @@ -70,7 +68,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, secst *sector.Store, commt *commitment.Tracker) (*Miner, error) { +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, secst *sector.Store) (*Miner, error) { return &Miner{ api: api, @@ -78,7 +76,6 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto h: h, ds: ds, secst: secst, - commt: commt, }, nil } @@ -214,10 +211,6 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal m.beginPosting(ctx) }() - if err := m.commt.TrackCommitSectorMsg(m.maddr, sinfo.SectorID, smsg.Cid()); err != nil { - return xerrors.Errorf("tracking sector commitment: %w", err) - } - return nil } From dd9d6c4ef02f50d04b71981d9fb60f1d2d555d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Oct 2019 18:40:48 +0100 Subject: [PATCH 083/230] Update sectorbuilder, v15 params --- build/proof-params/parameters.json | 64 +++++++++++++++--------------- extern/go-sectorbuilder | 2 +- go.mod | 1 - go.sum | 7 +++- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json index 0460fb226..e12686127 100644 --- a/build/proof-params/parameters.json +++ b/build/proof-params/parameters.json @@ -1,82 +1,82 @@ { - "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { + "v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": { "cid": "QmT22f1Np1GpW29NXD7Zrv3Ae4poMYhmkDjyscqL8QrJXY", "digest": "989fd8d989e0f7f1fe21bb010cf1b231", "sector_size": 16777216 }, - "v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { + "v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": { "cid": "QmVqSdc23to4UwduCCb25223rpSccvtcgPMfRKY1qjucDc", "digest": "c6d258c37243b8544238a98100e3e399", "sector_size": 16777216 }, - "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { + "v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": { "cid": "QmRTCqgokEGTMfWVaSr7qFXTNotmpd2QBEi8RsvSQKmPLz", "digest": "ff77a5e270afc6e1c7fbc19e48348fac", "sector_size": 1073741824 }, - "v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { + "v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": { "cid": "QmRssVAXRN3xp9VdSpTq1pNjkob3QiikoFZiM5hqrmh1VU", "digest": "b41f35ac26224258e366327716a835a4", "sector_size": 1073741824 }, - "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { + "v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": { "cid": "QmYNVRVzjXkuxJfnHTU5vmEcUBQf8dabXZ4m53SzqMkBv5", "digest": "d156b685e4a1fe3a1f7230b6a39b5ad4", "sector_size": 1024 }, - "v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { + "v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": { "cid": "QmaCEcsCFVuepMKdC5WURbr5ucEyLMNGxQaB7HqSnr2KGh", "digest": "06ff067ac78cdab5d7bbc82170882241", "sector_size": 1024 }, - "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { + "v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": { "cid": "QmVuabRvJ797NwLisGKwRURASGxopBBgg4rfNsbZoSYzAc", "digest": "0e1ceb79a459a60508f480e5b1fed7ac", "sector_size": 268435456 }, - "v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { + "v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": { "cid": "QmdWENZBAbuUty1vVNn9vmvj1XbJ5UC8qzpcVD35s5AJxG", "digest": "1b755c74b9d6823c014f6a7ef76249f2", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": { - "cid": "QmPG38HmDNVFiQJskqKe9sfSjyHfvZRihcfry78rt22FDT", - "digest": "8ea0b47e72250d5d6dab5d4f859e65de", + "v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": { + "cid": "QmZDVpWTw5Eti5pE7N5z1Cmqsw8hPXhUcvG3cQuceK56LH", + "digest": "6aa80306018ea1328f2d6faf8c080734", "sector_size": 16777216 }, - "v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": { - "cid": "Qmd3pNM22pgAoRT24tNyEZmeEWK2GtoZznBvzjie2YgqCn", - "digest": "e39f344757c919ae6bbc9b61311c73b2", + "v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": { + "cid": "QmaoXV7iVSJcfZ5qubYy7NBcXDSdnTzxH85d7M4bdDtfGZ", + "digest": "f6832eb736faf2960e920d32e9780b12", "sector_size": 16777216 }, - "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": { - "cid": "QmQsS6RqWmgdwPnHCwhBJH3WDPcAxhKfbQUs2bwa8D9su8", - "digest": "09879a69abcc51de5c1095f347c84e2b", + "v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": { + "cid": "QmbUW3a3q5DHBb7Ufk8iSbnSCZgbwpe3serqfwKmcTd11w", + "digest": "64024e461b07c869df2463d33dd28035", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": { - "cid": "QmSFBL5rg2TJv8QjLzrbr4c2KV2uDNN13RBVNUgwemJgM1", - "digest": "db0f245f7e9989879d2fa6328bd57d32", + "v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": { + "cid": "Qme3QgBBE7hUgrK7G9ZfJhzkbvViN5HALFpFduYs5K1piv", + "digest": "32496f4dc434b0ed9ef49cb62497a7d1", "sector_size": 268435456 }, - "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": { - "cid": "QmZ9UVBfviaNFsKyazA4k8GZcM1tHNDpDyrEK9qkxaMJXx", - "digest": "402a7c7c82eaa4af9fba3c7e4402b65b", + "v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": { + "cid": "QmZzgJmb8WXYDKxS22HDgnoBYcZzXDC7s2c2zsV7kouNZ9", + "digest": "cd91f7ccb2ff57a06f3375946dcbdc68", "sector_size": 1073741824 }, - "v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": { - "cid": "QmNmDcPVJ1bFFDcNCcRnEoQ6vNDNpKadLPHyEpUoF47gxV", - "digest": "2741c456346a3758e88249d1f4c0d227", + "v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": { + "cid": "QmRUMVzFnENbvyNb6aN2AJ2dnnewr1ESGA1UQLMVZZdsJM", + "digest": "92fc84b76dbe69c731518aebcb82ac82", "sector_size": 1073741824 }, - "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": { - "cid": "QmNqgCv6UjdKDNMuiwDweZ22TQYMd3gV6nWiA5PjMtNDVu", - "digest": "a9d316d0dbca152e653d41ad8e40058a", + "v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": { + "cid": "QmSixsGkxJXTAuFfWKy5aEXEDJEnpcb1GkdQVF8TCPWoHy", + "digest": "f8339ae93478ded3840d0bc7efa19953", "sector_size": 1024 }, - "v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": { - "cid": "QmTGQGThNFEjaMFgP32YLubACrtpkRoeVEfhDaWi5g6w8u", - "digest": "021b3e81e2980a50fd1ac07424d29a8d", + "v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": { + "cid": "QmTMC8hdZ2TkZ9BFuzHzRLM9SuR2PQdUrSAwABeCuHyV2f", + "digest": "f27f08ce1246ee6612c250bb12803ef1", "sector_size": 1024 } } \ No newline at end of file diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 692725ff2..e8231db9d 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 692725ff21919ce9c9df9ea87621b0c1e6a9746c +Subproject commit e8231db9dc64e79f7049e25c538b77b88eef54dc diff --git a/go.mod b/go.mod index b8eec445f..67f47edc4 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,6 @@ require ( go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect - golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect golang.org/x/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 google.golang.org/api v0.9.0 // indirect diff --git a/go.sum b/go.sum index 3d9bc15a9..796ce0e24 100644 --- a/go.sum +++ b/go.sum @@ -445,6 +445,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -613,9 +614,9 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= @@ -714,6 +715,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -749,6 +751,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -801,6 +804,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -817,6 +821,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 7fa3e2db7a0dde153a099e6d9b0a32da24413de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Oct 2019 19:10:29 +0100 Subject: [PATCH 084/230] WIP Interactive PoRep --- chain/actors/actor_miner.go | 9 ++++--- go.sum | 2 ++ lib/sectorbuilder/sectorbuilder.go | 33 ++++++++++++------------- lib/sectorbuilder/sectorbuilder_test.go | 2 ++ node/modules/storageminer.go | 2 ++ storage/miner.go | 4 ++- storage/sector/store.go | 14 +++++------ 7 files changed, 38 insertions(+), 28 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 28cc979a1..165c5ecd1 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -304,8 +304,11 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC // TODO: ensure normalization to ID address maddr := vmctx.Message().To + var pieces []sectorbuilder.PublicPieceInfo // TODO: GET ME FROM DEALS IN STORAGEMARKET + var seed []byte // TODO: GET ME FROM SOMEWHERE + rand, err := vmctx.GetRandomness(us.SubmitHeight + build.InteractivePoRepDelay) - if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, rand, params.Proof, params.SectorID); err != nil { + if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, rand, seed, params.Proof, params.SectorID, pieces); err != nil { return nil, err } else if !ok { return nil, aerrors.New(2, "bad proof!") @@ -579,8 +582,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof []byte, sectorID uint64) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, sectorID, proof) +func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64, pieces []sectorbuilder.PublicPieceInfo) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof, pieces) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } diff --git a/go.sum b/go.sum index 796ce0e24..e33480571 100644 --- a/go.sum +++ b/go.sum @@ -715,6 +715,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -751,6 +752,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index a2e4499b7..033372876 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -26,7 +26,11 @@ type SectorInfo = sectorbuilder.SectorInfo type SealTicket = sectorbuilder.SealTicket -type SealedSectorMetadata = sectorbuilder.SealedSectorMetadata +type SealSeed = sectorbuilder.SealSeed + +type SealCommitOutput = sectorbuilder.SealCommitOutput + +type PublicPieceInfo = sectorbuilder.PublicPieceInfo const CommLen = sectorbuilder.CommitmentBytesLen @@ -37,6 +41,7 @@ type SectorBuilder struct { type SectorBuilderConfig struct { SectorSize uint64 Miner address.Address + CacheDir string SealedDir string StagedDir string MetadataDir string @@ -44,8 +49,9 @@ type SectorBuilderConfig struct { func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) + nemWorkerThreads := uint8(5) // TODO: from config - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, nemWorkerThreads) if err != nil { return nil, err } @@ -84,12 +90,12 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, err return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } -func (sb *SectorBuilder) SealSector(sectorID uint64, ticket SealTicket) (SealedSectorMetadata, error) { - return sectorbuilder.SealSector(sb.handle, sectorID, ticket) +func (sb *SectorBuilder) SealSector(sectorID uint64, seed SealSeed) (SealCommitOutput, error) { + return sectorbuilder.SealCommit(sb.handle, sectorID, seed) } -func (sb *SectorBuilder) ResumeSealSector(sectorID uint64) (SealedSectorMetadata, error) { - return sectorbuilder.ResumeSealSector(sb.handle, sectorID) +func (sb *SectorBuilder) ResumeSealCommit(sectorID uint64) (SealCommitOutput, error) { + return sectorbuilder.ResumeSealCommit(sb.handle, sectorID) } func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { @@ -122,22 +128,15 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector -func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, sectorID uint64, proof []byte) (bool, error) { - var commRa, commDa, ticketa [32]byte +func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte, pieces []PublicPieceInfo) (bool, error) { + var commRa, commDa, ticketa, seeda [32]byte copy(commRa[:], commR) copy(commDa[:], commD) copy(ticketa[:], ticket) + copy(seeda[:], seed) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, sectorID, proof) -} - -func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP []byte, commD []byte, proof []byte) (bool, error) { - var commPa, commDa [32]byte - copy(commPa[:], commP) - copy(commDa[:], commD) - - return sectorbuilder.VerifyPieceInclusionProof(sectorSize, pieceSize, commPa, commDa, proof) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, pieces) } func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo { diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index b40a0b9cd..a4a2f7c15 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -36,12 +36,14 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } + cache := filepath.Join(dir, "cache") metadata := filepath.Join(dir, "meta") sealed := filepath.Join(dir, "sealed") staging := filepath.Join(dir, "staging") sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ SectorSize: sectorSize, + CacheDir:cache, SealedDir: sealed, StagedDir: staging, MetadataDir: metadata, diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 81b34522a..22773c6a2 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -55,6 +55,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod return nil, err } + cache := filepath.Join(sp, "cache") metadata := filepath.Join(sp, "meta") sealed := filepath.Join(sp, "sealed") staging := filepath.Join(sp, "staging") @@ -62,6 +63,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod sb := §orbuilder.SectorBuilderConfig{ Miner: minerAddr, SectorSize: ssize, + CacheDir:cache, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, diff --git a/storage/miner.go b/storage/miner.go index 089a46a3e..15fa17c88 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -92,7 +92,7 @@ func (m *Miner) Run(ctx context.Context) error { } func (m *Miner) commitUntrackedSectors(ctx context.Context) error { - sealed, err := m.secst.Sealed() + sealed, err := m.secst.Commited() if err != nil { return err } @@ -159,6 +159,8 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal return xerrors.Errorf("failed to check out own sector size: %w", err) } + // TODO: Interactive porep + ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], m.maddr, sinfo.Ticket.TicketBytes[:], sinfo.SectorID, sinfo.Proof) if err != nil { log.Error("failed to verify seal we just created: ", err) diff --git a/storage/sector/store.go b/storage/sector/store.go index 30363ee67..1073329f4 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -32,7 +32,7 @@ type dealMapping struct { Committed bool } -type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) +type TicketFn func(context.Context) (*sectorbuilder.SealSeed, error) // TODO: eventually handle sector storage here instead of in rust-sectorbuilder type Store struct { @@ -65,7 +65,7 @@ func (s *Store) Service() { go s.service() } -func (s *Store) poll() { +func (s *Store) poll() { // TODO: REMOVE ME (and just use the fact that sectorbuilder methods are now blocking) log.Debug("polling for sealed sectors...") // get a list of sectors to poll @@ -87,7 +87,7 @@ func (s *Store) poll() { continue } - if status.State == sealing_state.Sealed { + if status.State == sealing_state.Committed { done = append(done, status) } } @@ -119,7 +119,7 @@ func (s *Store) restartSealing() { return } - if status.State != sealing_state.Paused { + if status.State != sealing_state.CommittingPaused { // TODO: Also handle PreCommit! continue } @@ -127,7 +127,7 @@ func (s *Store) restartSealing() { go func() { // TODO: when we refactor wait-for-seal below, care about this output too // (see SealSector below) - _, err := s.sb.ResumeSealSector(sid) + _, err := s.sb.ResumeSealCommit(sid) if err != nil { return } @@ -293,7 +293,7 @@ func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.Sect return s.sb.SealStatus(sector) } -func (s *Store) Sealed() ([]sectorbuilder.SectorSealingStatus, error) { +func (s *Store) Commited() ([]sectorbuilder.SectorSealingStatus, error) { l, err := s.sb.GetAllStagedSectors() if err != nil { return nil, err @@ -306,7 +306,7 @@ func (s *Store) Sealed() ([]sectorbuilder.SectorSealingStatus, error) { return nil, err } - if status.State != sealing_state.Sealed { + if status.State != sealing_state.Committed { continue } out = append(out, status) From 407183b25896709d812ec0ae90e53d0f83e11c73 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 30 Oct 2019 23:33:31 +0100 Subject: [PATCH 085/230] Fix double locking License: MIT Signed-off-by: Jakub Sztandera --- node/builder.go | 71 +++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/node/builder.go b/node/builder.go index f704bef19..d082ea635 100644 --- a/node/builder.go +++ b/node/builder.go @@ -294,68 +294,57 @@ func ConfigFullNode(cfg *config.FullNode) Option { ) } -func repoFull(r repo.Repo) Option { - lr, err := r.Lock(repo.RepoFullNode) - if err != nil { - return Error(err) - } - c, err := lr.Config() - if err != nil { - return Error(err) - } +func configFull(c interface{}) Option { cfg, ok := c.(*config.FullNode) if !ok { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - return Options( - ConfigFullNode(cfg), - Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing - ) + return ConfigFullNode(cfg) } -func repoMiner(r repo.Repo) Option { - lr, err := r.Lock(repo.RepoStorageMiner) - if err != nil { - return Error(err) - } - c, err := lr.Config() - if err != nil { - return Error(err) - } - +func configMiner(c interface{}) Option { cfg, ok := c.(*config.StorageMiner) if !ok { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - return Options( - ConfigCommon(&cfg.Common), - Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing - ) + return ConfigCommon(&cfg.Common) } func Repo(r repo.Repo) Option { + return func(settings *Settings) error { + lr, err := r.Lock(settings.nodeType) + if err != nil { + return err + } + c, err := lr.Config() + if err != nil { + return err + } - return Options( - ApplyIf(isType(repo.RepoFullNode), repoFull(r)), - ApplyIf(isType(repo.RepoStorageMiner), repoMiner(r)), + return Options( + Override(new(repo.LockedRepo), modules.LockedRepo(lr)), // module handles closing - Override(new(dtypes.MetadataDS), modules.Datastore), - Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), + ApplyIf(isType(repo.RepoFullNode), configFull(c)), + ApplyIf(isType(repo.RepoStorageMiner), configMiner(c)), - Override(new(dtypes.ClientFilestore), modules.ClientFstore), - Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), - Override(new(dtypes.ClientDAG), modules.ClientDAG), + Override(new(dtypes.MetadataDS), modules.Datastore), + Override(new(dtypes.ChainBlockstore), modules.ChainBlockstore), - Override(new(ci.PrivKey), lp2p.PrivKey), - Override(new(ci.PubKey), ci.PrivKey.GetPublic), - Override(new(peer.ID), peer.IDFromPublicKey), + Override(new(dtypes.ClientFilestore), modules.ClientFstore), + Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore), + Override(new(dtypes.ClientDAG), modules.ClientDAG), - Override(new(types.KeyStore), modules.KeyStore), + Override(new(ci.PrivKey), lp2p.PrivKey), + Override(new(ci.PubKey), ci.PrivKey.GetPublic), + Override(new(peer.ID), peer.IDFromPublicKey), - Override(new(*dtypes.APIAlg), modules.APISecret), - ) + Override(new(types.KeyStore), modules.KeyStore), + + Override(new(*dtypes.APIAlg), modules.APISecret), + )(settings) + } } func FullAPI(out *api.FullNode) Option { From 6c4537f6a85417ab68ea91009c3315b7415390da Mon Sep 17 00:00:00 2001 From: hans Date: Mon, 28 Oct 2019 17:19:58 +0800 Subject: [PATCH 086/230] fix NewTipSet --- chain/types/tipset.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 3733a6f09..40b2b2b0b 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -74,9 +74,15 @@ func NewTipSet(blks []*BlockHeader) (*TipSet, error) { if b.Height != blks[0].Height { return nil, fmt.Errorf("cannot create tipset with mismatching heights") } + + for i, cid := range b.Parents { + if cid != blks[0].Parents[i] { + return nil, fmt.Errorf("cannot create tipset with mismatching parents") + } + } + ts.cids = append(ts.cids, b.Cid()) - // TODO: ensure the same parents } ts.height = blks[0].Height From 759094198c519d8ee7b9cbf4900088ce2e576ef0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 30 Oct 2019 18:22:50 -0700 Subject: [PATCH 087/230] Some more progress on interactive porep --- chain/actors/actor_miner.go | 25 +++++++-- chain/actors/actor_storagemarket.go | 40 ++++++++++++++- chain/deals/state_store.go | 1 + gen/main.go | 1 + go.sum | 1 + lib/sectorbuilder/sectorbuilder.go | 10 +++- node/modules/storageminer.go | 2 +- storage/miner.go | 78 ++++++++++++++++++++++++----- storage/sector/store.go | 71 +++----------------------- 9 files changed, 144 insertions(+), 85 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 165c5ecd1..fd70f0af0 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -98,6 +98,7 @@ type UnprovenSector struct { CommD []byte CommR []byte SubmitHeight uint64 + TicketEpoch uint64 } type StorageMinerConstructorParams struct { @@ -216,7 +217,6 @@ type SectorPreCommitInfo struct { CommR []byte Epoch uint64 - Proof []byte SectorNumber uint64 } @@ -227,6 +227,14 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon return nil, err } + if params.Epoch >= vmctx.BlockHeight() { + return nil, aerrors.New(1, "sector commitment must be based off past randomness") + } + + if vmctx.BlockHeight()-params.Epoch > 1000 { + return nil, aerrors.New(2, "sector commitment must be recent enough") + } + mi, err := loadMinerInfo(vmctx, self) if err != nil { return nil, err @@ -258,6 +266,7 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon CommR: params.CommR, CommD: params.CommD, SubmitHeight: vmctx.BlockHeight(), + TicketEpoch: params.Epoch, } nstate, err := vmctx.Storage().Put(self) @@ -305,10 +314,18 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC maddr := vmctx.Message().To var pieces []sectorbuilder.PublicPieceInfo // TODO: GET ME FROM DEALS IN STORAGEMARKET - var seed []byte // TODO: GET ME FROM SOMEWHERE - rand, err := vmctx.GetRandomness(us.SubmitHeight + build.InteractivePoRepDelay) - if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, rand, seed, params.Proof, params.SectorID, pieces); err != nil { + ticket, err := vmctx.GetRandomness(us.TicketEpoch) + if err != nil { + return nil, aerrors.Wrap(err, "failed to get ticket randomness") + } + + seed, err := vmctx.GetRandomness(us.SubmitHeight + build.InteractivePoRepDelay) + if err != nil { + return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") + } + + if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, ticket, params.Proof, seed, params.SectorID, pieces); err != nil { return nil, err } else if !ok { return nil, aerrors.New(2, "bad proof!") diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 9c3fcb9aa..798818c17 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -29,9 +29,10 @@ type smaMethods struct { SlashStorageDealCollateral uint64 GetLastExpirationFromDealIDs uint64 ActivateStorageDeals uint64 + ComputeDataCommitment uint64 } -var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} +var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} func (sma StorageMarketActor) Exports() []interface{} { return []interface{}{ @@ -45,6 +46,7 @@ func (sma StorageMarketActor) Exports() []interface{} { // 9: sma.SlashStorageDealCollateral, // 10: sma.GetLastExpirationFromDealIDs, 11: sma.ActivateStorageDeals, // TODO: move under PublishStorageDeals after specs team approves + 12: sma.ComputeDataCommitment, } } @@ -585,6 +587,42 @@ func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) { return types.BigSub(from, amt), types.BigAdd(to, amt) } +var ComputeDataCommitmentParams struct { + DealIDs []uint64 + SectorSize uint64 +} + +func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx types.VMContext, params *ComputeDataCommitmentParams) ([]byte, ActorError) { + var self StorageMarketState + old := vmctx.Storage().GetHead() + if err := vmctx.Storage().Get(old, &self); err != nil { + return nil, err + } + + deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) + if err != nil { + // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal + return nil, aerrors.HandleExternalError(err, "loading deals amt") + } + + for _, deal := range params.DealIDs { + var dealInfo OnChainDeal + if err := deals.Get(deal, &dealInfo); err != nil { + if _, is := err.(*amt.ErrNotFound); is { + return nil, aerrors.New(3, "deal not found") + } + return nil, aerrors.HandleExternalError(err, "getting deal info failed") + } + + _ = dealInfo + } + + // TODO: rust-fil-proofs-magic + var commDoutput [32]byte + + return commDoutput[:], nil +} + /* func (sma StorageMarketActor) HandleCronAction(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) { diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go index ae959146f..cde6240be 100644 --- a/chain/deals/state_store.go +++ b/chain/deals/state_store.go @@ -2,6 +2,7 @@ package deals import ( "bytes" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" diff --git a/gen/main.go b/gen/main.go index 102872f08..a588dfadb 100644 --- a/gen/main.go +++ b/gen/main.go @@ -88,6 +88,7 @@ func main() { actors.ActivateStorageDealsParams{}, actors.ProcessStorageDealsPaymentParams{}, actors.OnChainDeal{}, + actors.ComputeDataCommitmentParams{}, ) if err != nil { fmt.Println(err) diff --git a/go.sum b/go.sum index e33480571..76f9b1a65 100644 --- a/go.sum +++ b/go.sum @@ -823,6 +823,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 033372876..0bcf19506 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -28,6 +28,8 @@ type SealTicket = sectorbuilder.SealTicket type SealSeed = sectorbuilder.SealSeed +type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput + type SealCommitOutput = sectorbuilder.SealCommitOutput type PublicPieceInfo = sectorbuilder.PublicPieceInfo @@ -41,7 +43,7 @@ type SectorBuilder struct { type SectorBuilderConfig struct { SectorSize uint64 Miner address.Address - CacheDir string + CacheDir string SealedDir string StagedDir string MetadataDir string @@ -90,7 +92,11 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, err return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } -func (sb *SectorBuilder) SealSector(sectorID uint64, seed SealSeed) (SealCommitOutput, error) { +func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket) (SealPreCommitOutput, error) { + return sectorbuilder.SealPreCommit(sb.handle, sectorID, ticket) +} + +func (sb *SectorBuilder) SealCommit(sectorID uint64, seed SealSeed) (SealCommitOutput, error) { return sectorbuilder.SealCommit(sb.handle, sectorID, seed) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 22773c6a2..73d55c5d1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -63,7 +63,7 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod sb := §orbuilder.SectorBuilderConfig{ Miner: minerAddr, SectorSize: ssize, - CacheDir:cache, + CacheDir: cache, MetadataDir: metadata, SealedDir: sealed, StagedDir: staging, diff --git a/storage/miner.go b/storage/miner.go index 15fa17c88..1ad4f3eea 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -12,6 +12,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/events" @@ -159,15 +160,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal return xerrors.Errorf("failed to check out own sector size: %w", err) } - // TODO: Interactive porep - - ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], m.maddr, sinfo.Ticket.TicketBytes[:], sinfo.SectorID, sinfo.Proof) - if err != nil { - log.Error("failed to verify seal we just created: ", err) - } - if !ok { - log.Error("seal we just created failed verification") - } + _ = ssize // TODO: 2 stage commit /*deals, err := m.secst.DealsForCommit(sinfo.SectorID) @@ -178,7 +171,6 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal params := &actors.SectorPreCommitInfo{ CommD: sinfo.CommD[:], CommR: sinfo.CommR[:], - Proof: sinfo.Proof, Epoch: sinfo.Ticket.BlockHeight, //DealIDs: deals, @@ -205,12 +197,74 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal } go func() { - _, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + // TODO: maybe just mark this down in the datastore and handle it differently? This feels complicated to restart + mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) if err != nil { return } - m.beginPosting(ctx) + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay + + err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { + go func() { + rand, err := m.api.ChainGetRandomness(ctx, ts, nil, ts.Height()-randHeight) + if err != nil { + log.Error(errors.Errorf("failed to get randomness for computing seal proof: %w", err)) + return + } + + // TODO: should this get scheduled to preserve proper resource consumption? + proof, err := m.secst.SealComputeProof(ctx, sinfo.SectorID, rand) + if err != nil { + log.Error(errors.Errorf("computing seal proof failed: %w", err)) + return + } + + params := &actors.SectorProveCommitInfo{ + Proof: proof, + SectorID: sinfo.SectorID, + //DealIDs: deals, + } + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + log.Errorf(errors.Wrap(aerr, "could not serialize commit sector parameters")) + return + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.ProveCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return errors.Wrap(err, "pushing message to mpool") + } + + // TODO: now wait for this to get included and handle errors? + _, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + log.Errorf("failed to wait for porep inclusion: %s", err) + return + } + + m.beginPosting(ctx) + }() + + return nil + }, func(ts *types.TipSet) error { + log.Warn("revert in interactive commit sector step") + return nil + }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) + if err != nil { + return + } }() return nil diff --git a/storage/sector/store.go b/storage/sector/store.go index 1073329f4..19460e8bb 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "sync" - "time" "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-datastore" @@ -65,48 +64,6 @@ func (s *Store) Service() { go s.service() } -func (s *Store) poll() { // TODO: REMOVE ME (and just use the fact that sectorbuilder methods are now blocking) - log.Debug("polling for sealed sectors...") - - // get a list of sectors to poll - s.waitingLk.Lock() - toPoll := make([]uint64, 0, len(s.waiting)) - - for id := range s.waiting { - toPoll = append(toPoll, id) - } - s.waitingLk.Unlock() - - var done []sectorbuilder.SectorSealingStatus - - // check status of each - for _, sec := range toPoll { - status, err := s.sb.SealStatus(sec) - if err != nil { - log.Errorf("getting seal status: %s", err) - continue - } - - if status.State == sealing_state.Committed { - done = append(done, status) - } - } - - // send updates - s.waitingLk.Lock() - for _, sector := range done { - watch, ok := s.waiting[sector.SectorID] - if ok { - close(watch) - delete(s.waiting, sector.SectorID) - } - for _, c := range s.incoming { - c <- sector // TODO: ctx! - } - } - s.waitingLk.Unlock() -} - func (s *Store) restartSealing() { sectors, err := s.sb.GetAllStagedSectors() if err != nil { @@ -135,26 +92,6 @@ func (s *Store) restartSealing() { } } -func (s *Store) service() { - poll := time.Tick(5 * time.Second) - - s.restartSealing() - - for { - select { - case <-poll: - s.poll() - case <-s.closeCh: - s.waitingLk.Lock() - for _, c := range s.incoming { - close(c) - } - s.waitingLk.Unlock() - return - } - } -} - func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { sectorID, err = s.sb.AddPiece(ref, size, r) if err != nil { @@ -234,7 +171,7 @@ func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { } } -func (s *Store) SealSector(ctx context.Context, sectorID uint64) error { +func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) error { tkt, err := s.tktFn(ctx) if err != nil { return err @@ -242,13 +179,17 @@ func (s *Store) SealSector(ctx context.Context, sectorID uint64) error { // TODO: That's not async, is it? // - If not then we probably can drop this wait-for-seal hack below - _, err = s.sb.SealSector(sectorID, *tkt) + _, err = s.sb.SealPreCommit(sectorID, *tkt) if err != nil { return err } return nil } +func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, rand []byte) ([]byte, error) { + panic("TODO") +} + func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { s.waitingLk.Lock() var at = -1 From 8cb14335c3220a047a5bc5a5cc700ca2c0d4ad29 Mon Sep 17 00:00:00 2001 From: wanghui Date: Thu, 31 Oct 2019 13:11:10 +0800 Subject: [PATCH 088/230] use reflect select --- lib/jsonrpc/client.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index cb144de77..126da7d04 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -174,15 +174,26 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) go func() { for buf.Len() > 0 { - select { - case <- chCtx.Done(): + front := buf.Front() + bufLk.Unlock() + + cases := []reflect.SelectCase{ + { + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(chCtx.Done()), + }, + { + Dir: reflect.SelectSend, + Chan: ch, + Send: front.Value.(reflect.Value).Elem(), + }, + } + + switch chosen, _, _ := reflect.Select(cases); chosen { + case 0: + bufLk.Lock() buf.Init() - default: - front := buf.Front() - bufLk.Unlock() - - ch.Send(front.Value.(reflect.Value).Elem()) - + case 1: bufLk.Lock() buf.Remove(front) } From 0aa56ba6659195062114e430cf1664dbc3fd3532 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 31 Oct 2019 13:36:33 +0800 Subject: [PATCH 089/230] Refactor the PowerCmp function Why we have this: This algorithm here directly follow the rule of ratio to win, and easier to understand. Just think about a smaller space, e.g. 100, instead of 2^256. When one miner has 10% power, it should have 10% ratio to win. that is, when the h is in {0..9}, it wins. Simply speaking, it is: (h + 1) / 100 <= 10% , instead of h / 99 < 10% The former is easier to understand, though both are equivalent when h is an integer and power_ratio < 1) --- chain/types/blockheader.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 1ede605d4..5fb72545b 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -166,24 +166,25 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { /* Need to check that - h(vrfout) / max(h) < e * minerPower / totalPower + (h(vrfout) + 1) / (max(h) + 1) <= e * minerPower / totalPower max(h) == 2^256-1 which in terms of integer math means: - h(vrfout) * totalPower < e * minerPower * (2^256-1) + (h(vrfout) + 1) * totalPower <= e * minerPower * 2^256 */ h := sha256.Sum256(eproof) lhs := BigFromBytes(h[:]).Int + lhs = lhs.Add(lhs, NewInt(1).Int) lhs = lhs.Mul(lhs, totpow.Int) - // rhs = minerPower * 2^256 - minerPower - // rhs = minerPower << 256 - minerPower + // rhs = minerPower * 2^256 + // rhs = minerPower << 256 rhs := new(big.Int).Lsh(mpow.Int, 256) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) - rhs = rhs.Sub(rhs, mpow.Int) - - return lhs.Cmp(rhs) == -1 + + // return true if lhs is less than or equal to rhs + return lhs.Cmp(rhs) < 1 } func (t *Ticket) Equals(ot *Ticket) bool { From 85854578d1ea622bfa21b195bee006fc78e0bf77 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 31 Oct 2019 16:44:26 +0800 Subject: [PATCH 090/230] Update chain/types/blockheader.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Łukasz Magiera --- chain/types/blockheader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 5fb72545b..1a1d5b875 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -184,7 +184,7 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { rhs = rhs.Mul(rhs, blocksPerEpoch.Int) // return true if lhs is less than or equal to rhs - return lhs.Cmp(rhs) < 1 + return lhs.Cmp(rhs) <= 0 } func (t *Ticket) Equals(ot *Ticket) bool { From 2aa8eebb993e36712141d4a6b80d5f88c9d5a86f Mon Sep 17 00:00:00 2001 From: wanghui Date: Thu, 31 Oct 2019 17:39:42 +0800 Subject: [PATCH 091/230] move select --- lib/jsonrpc/client.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 126da7d04..81cf4ec0a 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -189,12 +189,13 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) }, } - switch chosen, _, _ := reflect.Select(cases); chosen { + chosen, _, _ := reflect.Select(cases) + bufLk.Lock() + + switch chosen { case 0: - bufLk.Lock() buf.Init() case 1: - bufLk.Lock() buf.Remove(front) } } From a51cbe408167c982e22829bbad8667fee3e6ee8f Mon Sep 17 00:00:00 2001 From: yaohcn Date: Thu, 31 Oct 2019 17:41:11 +0800 Subject: [PATCH 092/230] specific ipfs gateway --- build/paramfetch.go | 14 +++++++------- cli/params.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/main.go | 5 +++++ cmd/lotus-storage-miner/run.go | 2 +- cmd/lotus/daemon.go | 5 +++-- cmd/lotus/main.go | 5 +++++ lib/sectorbuilder/sectorbuilder_test.go | 2 +- 8 files changed, 24 insertions(+), 13 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index abfeca08b..97ae76d74 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -21,7 +21,7 @@ import ( var log = logging.Logger("build") //const gateway = "http://198.211.99.118/ipfs/" -const gateway = "https://ipfs.io/ipfs/" +const GateWay = "https://ipfs.io/ipfs/" const paramdir = "/var/tmp/filecoin-proof-parameters" type paramFile struct { @@ -37,7 +37,7 @@ type fetch struct { errs []error } -func GetParams(storage bool) error { +func GetParams(storage bool, gateway string) error { if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) { return err } @@ -59,13 +59,13 @@ func GetParams(storage bool) error { continue } - ft.maybeFetchAsync(name, info) + ft.maybeFetchAsync(name, info, gateway) } return ft.wait() } -func (ft *fetch) maybeFetchAsync(name string, info paramFile) { +func (ft *fetch) maybeFetchAsync(name string, info paramFile, gateway string) { ft.wg.Add(1) go func() { @@ -84,7 +84,7 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { ft.fetchLk.Lock() defer ft.fetchLk.Unlock() - if err := doFetch(path, info); err != nil { + if err := doFetch(path, info, gateway); err != nil { ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s: %w", path, err)) } }() @@ -116,8 +116,8 @@ func (ft *fetch) wait() error { return multierr.Combine(ft.errs...) } -func doFetch(out string, info paramFile) error { - log.Infof("Fetching %s", out) +func doFetch(out string, info paramFile, gateway string) error { + log.Infof("Fetching %s from %s", out, gateway) resp, err := http.Get(gateway + info.Cid) if err != nil { diff --git a/cli/params.go b/cli/params.go index 6f6017ca3..8aadebe58 100644 --- a/cli/params.go +++ b/cli/params.go @@ -10,7 +10,7 @@ var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", Action: func(cctx *cli.Context) error { - if err := build.GetParams(true); err != nil { + if err := build.GetParams(true, cctx.String("gateway")); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 54d488a1b..a6d5e7527 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -59,7 +59,7 @@ var initCmd = &cli.Command{ log.Info("Initializing lotus storage miner") log.Info("Checking proof parameters") - if err := build.GetParams(true); err != nil { + if err := build.GetParams(true, cctx.String("gateway")); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index b6aad1d73..d6a2afac1 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -65,6 +65,11 @@ func main() { EnvVars: []string{"LOTUS_STORAGE_PATH"}, Value: "~/.lotusstorage", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "gateway", + EnvVars: []string{"GATE_WAY"}, + Value: build.GateWay, + }, }, Commands: append(local, lcli.Commands...), diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index d0806eb16..717047bbd 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -34,7 +34,7 @@ var runCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if err := build.GetParams(true); err != nil { + if err := build.GetParams(true, cctx.String("gateway")); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c9b736c72..1a08320e3 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -4,9 +4,10 @@ package main import ( "context" - "github.com/filecoin-project/lotus/peermgr" "io/ioutil" + "github.com/filecoin-project/lotus/peermgr" + "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -57,7 +58,7 @@ var DaemonCmd = &cli.Command{ return err } - if err := build.GetParams(false); err != nil { + if err := build.GetParams(false, cctx.String("gateway")); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index 38fc478ac..f6797a43d 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -55,6 +55,11 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "gateway", + EnvVars: []string{"GATE_WAY"}, + Value: build.GateWay, + }, }, Commands: append(local, lcli.Commands...), diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index b40a0b9cd..92cac305d 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -22,7 +22,7 @@ func TestSealAndVerify(t *testing.T) { t.Skip("this is slow") build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true); err != nil { + if err := build.GetParams(true, build.GateWay); err != nil { t.Fatal(err) } From f35c8fe4268e2e8d8aeddc631a10db0fe10782bf Mon Sep 17 00:00:00 2001 From: steven004 Date: Thu, 31 Oct 2019 18:11:57 +0800 Subject: [PATCH 093/230] No AdjustmentPeriod anyu more --- build/params.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/build/params.go b/build/params.go index e810ff45f..699a21399 100644 --- a/build/params.go +++ b/build/params.go @@ -97,9 +97,6 @@ const FilecoinPrecision = 1000000000000000000 // Blocks const HalvingPeriodBlocks = 6 * 365 * 24 * 60 * 2 -// Blocks -const AdjustmentPeriod = 7 * 24 * 60 * 2 - // TODO: Move other important consts here func init() { From 9c57ca03d97c9bff464cc7cb7da2939c340d4635 Mon Sep 17 00:00:00 2001 From: yaohcn Date: Thu, 31 Oct 2019 18:21:37 +0800 Subject: [PATCH 094/230] use IPFS_GATEWAY --- build/paramfetch.go | 20 ++++++++++++-------- cli/params.go | 2 +- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/main.go | 5 ----- cmd/lotus-storage-miner/run.go | 2 +- cmd/lotus/daemon.go | 2 +- cmd/lotus/main.go | 5 ----- lib/sectorbuilder/sectorbuilder_test.go | 2 +- 8 files changed, 17 insertions(+), 23 deletions(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index 97ae76d74..39e6ffda6 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -21,7 +21,7 @@ import ( var log = logging.Logger("build") //const gateway = "http://198.211.99.118/ipfs/" -const GateWay = "https://ipfs.io/ipfs/" +const gateway = "https://ipfs.io/ipfs/" const paramdir = "/var/tmp/filecoin-proof-parameters" type paramFile struct { @@ -37,7 +37,7 @@ type fetch struct { errs []error } -func GetParams(storage bool, gateway string) error { +func GetParams(storage bool) error { if err := os.Mkdir(paramdir, 0755); err != nil && !os.IsExist(err) { return err } @@ -59,13 +59,13 @@ func GetParams(storage bool, gateway string) error { continue } - ft.maybeFetchAsync(name, info, gateway) + ft.maybeFetchAsync(name, info) } return ft.wait() } -func (ft *fetch) maybeFetchAsync(name string, info paramFile, gateway string) { +func (ft *fetch) maybeFetchAsync(name string, info paramFile) { ft.wg.Add(1) go func() { @@ -84,7 +84,7 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile, gateway string) { ft.fetchLk.Lock() defer ft.fetchLk.Unlock() - if err := doFetch(path, info, gateway); err != nil { + if err := doFetch(path, info); err != nil { ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s: %w", path, err)) } }() @@ -116,10 +116,14 @@ func (ft *fetch) wait() error { return multierr.Combine(ft.errs...) } -func doFetch(out string, info paramFile, gateway string) error { - log.Infof("Fetching %s from %s", out, gateway) +func doFetch(out string, info paramFile) error { + gw := os.Getenv("IPFS_GATEWAY") + if gw == "" { + gw = gateway + } + log.Infof("Fetching %s from %s", out, gw) - resp, err := http.Get(gateway + info.Cid) + resp, err := http.Get(gw + info.Cid) if err != nil { return err } diff --git a/cli/params.go b/cli/params.go index 8aadebe58..6f6017ca3 100644 --- a/cli/params.go +++ b/cli/params.go @@ -10,7 +10,7 @@ var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", Action: func(cctx *cli.Context) error { - if err := build.GetParams(true, cctx.String("gateway")); err != nil { + if err := build.GetParams(true); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a6d5e7527..54d488a1b 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -59,7 +59,7 @@ var initCmd = &cli.Command{ log.Info("Initializing lotus storage miner") log.Info("Checking proof parameters") - if err := build.GetParams(true, cctx.String("gateway")); err != nil { + if err := build.GetParams(true); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index d6a2afac1..b6aad1d73 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -65,11 +65,6 @@ func main() { EnvVars: []string{"LOTUS_STORAGE_PATH"}, Value: "~/.lotusstorage", // TODO: Consider XDG_DATA_HOME }, - &cli.StringFlag{ - Name: "gateway", - EnvVars: []string{"GATE_WAY"}, - Value: build.GateWay, - }, }, Commands: append(local, lcli.Commands...), diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 717047bbd..d0806eb16 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -34,7 +34,7 @@ var runCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - if err := build.GetParams(true, cctx.String("gateway")); err != nil { + if err := build.GetParams(true); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 1a08320e3..62b579eb1 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -58,7 +58,7 @@ var DaemonCmd = &cli.Command{ return err } - if err := build.GetParams(false, cctx.String("gateway")); err != nil { + if err := build.GetParams(false); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } diff --git a/cmd/lotus/main.go b/cmd/lotus/main.go index f6797a43d..38fc478ac 100644 --- a/cmd/lotus/main.go +++ b/cmd/lotus/main.go @@ -55,11 +55,6 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, - &cli.StringFlag{ - Name: "gateway", - EnvVars: []string{"GATE_WAY"}, - Value: build.GateWay, - }, }, Commands: append(local, lcli.Commands...), diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 92cac305d..b40a0b9cd 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -22,7 +22,7 @@ func TestSealAndVerify(t *testing.T) { t.Skip("this is slow") build.SectorSizes = []uint64{sectorSize} - if err := build.GetParams(true, build.GateWay); err != nil { + if err := build.GetParams(true); err != nil { t.Fatal(err) } From a917ffee16d8e1efde4134bab447c1208745bccf Mon Sep 17 00:00:00 2001 From: steven004 Date: Thu, 31 Oct 2019 18:54:13 +0800 Subject: [PATCH 095/230] minor code change, no function impact --- build/params.go | 5 +++++ chain/store/weight.go | 14 ++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/build/params.go b/build/params.go index 699a21399..d6453f6f5 100644 --- a/build/params.go +++ b/build/params.go @@ -97,6 +97,11 @@ const FilecoinPrecision = 1000000000000000000 // Blocks const HalvingPeriodBlocks = 6 * 365 * 24 * 60 * 2 +// constants for Weight calculation +// The ratio of weight contributed by short-term vs long-term factors in a given round +const WRatioNum = int64(1) +const WRatioDen = 2 + // TODO: Move other important consts here func init() { diff --git a/chain/store/weight.go b/chain/store/weight.go index 7c1f80ceb..3f52112cb 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -11,6 +11,8 @@ import ( ) var zero = types.NewInt(0) +var wRatio_num = build.WRatioNum +var wRatio_den = build.WRatioDen func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigInt, error) { if ts == nil { @@ -37,17 +39,17 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn tpow := types.BigFromBytes(ret.Return) if tpow.GreaterThan(zero) { log2P = int64(tpow.BitLen() - 1) + } else { + // Not really expect to be here ... + return types.EmptyInt, xerrors.Errorf("All power in the net is gone. The net is dead!") } - out.Add(out, big.NewInt(log2P*256)) + out.Add(out, big.NewInt(log2P << 8)) // (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den) - wRatioNum := int64(1) - wRatioDen := 2 - - eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * wRatioNum) << 8) - eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*wRatioDen))) + eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * wRatio_num) << 8) + eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch * wRatio_den))) out.Add(out, eWeight) return types.BigInt{Int: out}, nil From 4e478330a42c54b2a87aab60197ef9631e9d51ec Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 31 Oct 2019 09:55:35 -0700 Subject: [PATCH 096/230] more refactoring for interactive porep scheduling --- chain/actors/actor_miner.go | 21 ++- chain/actors/actor_storagemarket.go | 2 +- chain/actors/cbor_gen.go | 132 +++++++++++++---- chain/deals/provider.go | 8 +- chain/deals/provider_states.go | 60 ++++---- chain/types/cbor_gen.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 4 +- lib/sectorbuilder/sectorbuilder_test.go | 38 ++--- storage/miner.go | 180 +----------------------- storage/sector/store.go | 44 ++---- 10 files changed, 191 insertions(+), 300 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index fd70f0af0..e18fda36c 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -313,8 +313,6 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC // TODO: ensure normalization to ID address maddr := vmctx.Message().To - var pieces []sectorbuilder.PublicPieceInfo // TODO: GET ME FROM DEALS IN STORAGEMARKET - ticket, err := vmctx.GetRandomness(us.TicketEpoch) if err != nil { return nil, aerrors.Wrap(err, "failed to get ticket randomness") @@ -325,7 +323,20 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") } - if ok, err := ValidatePoRep(maddr, mi.SectorSize, us.CommD, us.CommR, ticket, params.Proof, seed, params.SectorID, pieces); err != nil { + enc, err := SerializeParams(&ComputeDataCommitmentParams{ + DealIDs: params.DealIDs, + SectorSize: mi.SectorSize, + }) + if err != nil { + return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams") + } + + commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc) + if err != nil { + return nil, aerrors.Wrap(err, "failed to compute data commitment") + } + + if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { return nil, aerrors.New(2, "bad proof!") @@ -599,8 +610,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } -func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64, pieces []sectorbuilder.PublicPieceInfo) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof, pieces) +func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { + ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof) if err != nil { return false, aerrors.Absorb(err, 25, "verify seal failed") } diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 798818c17..6eeeb7f99 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -587,7 +587,7 @@ func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) { return types.BigSub(from, amt), types.BigAdd(to, amt) } -var ComputeDataCommitmentParams struct { +type ComputeDataCommitmentParams struct { DealIDs []uint64 SectorSize uint64 } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 84de28117..63202da1a 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -542,7 +542,7 @@ func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{133}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } @@ -567,14 +567,6 @@ func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { - return err - } - if _, err := w.Write(t.Proof); err != nil { - return err - } - // t.t.SectorNumber (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorNumber)); err != nil { return err @@ -593,7 +585,7 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -641,23 +633,6 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.Epoch = extra - // t.t.Proof ([]uint8) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Proof = make([]byte, extra) - if _, err := io.ReadFull(br, t.Proof); err != nil { - return err - } // t.t.SectorNumber (uint64) maj, extra, err = cbg.CborReadHeader(br) @@ -676,7 +651,7 @@ func (t *UnprovenSector) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{131}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } @@ -700,6 +675,11 @@ func (t *UnprovenSector) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SubmitHeight)); err != nil { return err } + + // t.t.TicketEpoch (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TicketEpoch)); err != nil { + return err + } return nil } @@ -714,7 +694,7 @@ func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 3 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -762,6 +742,16 @@ func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.SubmitHeight = extra + // t.t.TicketEpoch (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.TicketEpoch = extra return nil } @@ -3697,3 +3687,87 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { t.ActivationEpoch = extra return nil } + +func (t *ComputeDataCommitmentParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + + // t.t.SectorSize (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + return err + } + return nil +} + +func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + // t.t.SectorSize (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = extra + return nil +} diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 356f7d2e7..132e3bf12 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storage" ) type MinerDeal struct { @@ -41,7 +41,7 @@ type Provider struct { ask *types.SignedStorageAsk askLk sync.Mutex - secst *sectorblocks.SectorBlocks + sminer *storage.Miner full api.FullNode // TODO: Use a custom protocol or graphsync in the future @@ -68,7 +68,7 @@ type minerDealUpdate struct { mut func(*MinerDeal) } -func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { +func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -79,7 +79,7 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, dag dty } h := &Provider{ - secst: secst, + sminer: sminer, dag: dag, full: fullNode, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 6c86eb8e9..c43628d13 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -4,7 +4,6 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" @@ -229,39 +228,48 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) if err != nil { return nil, err } + _ = pcid - sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.DealID) - if err != nil { - return nil, xerrors.Errorf("AddPiece failed: %s", err) - } + /* + sectorID, err := p.sminer.AddUnixfsPiece(pcid, uf, deal.DealID) + if err != nil { + return nil, xerrors.Errorf("AddPiece failed: %s", err) + } + log.Warnf("New Sector: %d", sectorID) - log.Warnf("New Sector: %d", sectorID) - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil + return func(deal *MinerDeal) { + deal.SectorID = sectorID + }, nil + */ + panic("fixme") } // SEALING func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) { - status, err := p.secst.WaitSeal(ctx, deal.SectorID) - if err != nil { - return sectorbuilder.SectorSealingStatus{}, err - } + panic("fixme") - switch status.State { - case sealing_state.Sealed: - case sealing_state.Failed: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) - case sealing_state.Pending: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) - case sealing_state.Sealing: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) - default: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) - } + /* + status, err := p.sminer.WaitSeal(ctx, deal.SectorID) + if err != nil { + return sectorbuilder.SectorSealingStatus{}, err + } + + switch status.State { + case sealing_state.Sealed: + case sealing_state.Failed: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) + case sealing_state.Pending: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) + case sealing_state.Sealing: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) + default: + return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) + } + + return status, nil + */ - return status, nil } func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { @@ -273,7 +281,7 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal log.Warnf("Sending deal response failed: %s", err) } - if err := p.secst.SealSector(ctx, deal.SectorID); err != nil { + if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { return nil, xerrors.Errorf("sealing sector failed: %w", err) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 9cf6ec913..ab63b28a1 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - "github.com/ipfs/go-cid" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 0bcf19506..0c1aeb66e 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -134,7 +134,7 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector -func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte, pieces []PublicPieceInfo) (bool, error) { +func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { var commRa, commDa, ticketa, seeda [32]byte copy(commRa[:], commR) copy(commDa[:], commD) @@ -142,7 +142,7 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address copy(seeda[:], seed) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, pieces) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, nil) } func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo { diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index a4a2f7c15..3c86bb6ef 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -1,19 +1,15 @@ package sectorbuilder_test import ( - "context" "io" "io/ioutil" "math/rand" "path/filepath" "testing" - "github.com/ipfs/go-datastore" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/storage/sector" ) const sectorSize = 1024 @@ -43,7 +39,7 @@ func TestSealAndVerify(t *testing.T) { sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ SectorSize: sectorSize, - CacheDir:cache, + CacheDir: cache, SealedDir: sealed, StagedDir: staging, MetadataDir: metadata, @@ -53,31 +49,35 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - // TODO: Consider fixing - store := sector.NewStore(sb, datastore.NewMapDatastore(), func(ctx context.Context) (*sectorbuilder.SealTicket, error) { - return §orbuilder.SealTicket{ - BlockHeight: 5, - TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, - }, nil - }) - - store.Service() - dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) - sid, err := store.AddPiece("foo", dlen, r) + sid, err := sb.AddPiece("foo", dlen, r) if err != nil { t.Fatal(err) } - if err := store.SealSector(context.TODO(), sid); err != nil { + ticket := sectorbuilder.SealTicket{ + BlockHeight: 5, + TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, + } + + pco, err := sb.SealPreCommit(sid, ticket) + if err != nil { t.Fatal(err) } - ssinfo := <-store.Incoming() + seed := sectorbuilder.SealSeed{ + BlockHeight: 15, + TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8}, + } - ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) + sco, err := sb.SealCommit(sid, seed) + if err != nil { + t.Fatal(err) + } + + ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], pco.CommD[:], addr, ticket.TicketBytes[:], seed.TicketBytes[:], sid, sco.Proof) if err != nil { t.Fatal(err) } diff --git a/storage/miner.go b/storage/miner.go index 1ad4f3eea..bb3be8f8c 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -9,11 +9,8 @@ import ( logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" @@ -87,187 +84,12 @@ func (m *Miner) Run(ctx context.Context) error { m.events = events.NewEvents(ctx, m.api) - go m.handlePostingSealedSectors(ctx) go m.beginPosting(ctx) return nil } -func (m *Miner) commitUntrackedSectors(ctx context.Context) error { - sealed, err := m.secst.Commited() - if err != nil { - return err - } - - chainSectors, err := m.api.StateMinerSectors(ctx, m.maddr, nil) - if err != nil { - return err - } - - onchain := map[uint64]struct{}{} - for _, chainSector := range chainSectors { - onchain[chainSector.SectorID] = struct{}{} - } - - for _, s := range sealed { - if _, ok := onchain[s.SectorID]; ok { - continue - } - - log.Warnf("Missing commitment for sector %d, committing sector", s.SectorID) - - if err := m.commitSector(ctx, s); err != nil { - log.Error("Committing uncommitted sector failed: ", err) - } - } - return nil -} - -func (m *Miner) handlePostingSealedSectors(ctx context.Context) { - incoming := m.secst.Incoming() - defer m.secst.CloseIncoming(incoming) - - if err := m.commitUntrackedSectors(ctx); err != nil { - log.Error(err) - } - - for { - select { - case sinfo, ok := <-incoming: - if !ok { - // TODO: set some state variable so that this state can be - // visible via some status command - log.Warn("sealed sector channel closed, aborting process") - return - } - - if err := m.commitSector(ctx, sinfo); err != nil { - log.Errorf("failed to commit sector: %s", err) - continue - } - - case <-ctx.Done(): - log.Warn("exiting seal posting routine") - return - } - } -} - func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { - log.Info("committing sector") - - ssize, err := m.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to check out own sector size: %w", err) - } - - _ = ssize - - // TODO: 2 stage commit - /*deals, err := m.secst.DealsForCommit(sinfo.SectorID) - if err != nil { - return xerrors.Errorf("getting sector deals failed: %w", err) - } - */ - params := &actors.SectorPreCommitInfo{ - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - Epoch: sinfo.Ticket.BlockHeight, - - //DealIDs: deals, - SectorNumber: sinfo.SectorID, - } - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - return errors.Wrap(aerr, "could not serialize commit sector parameters") - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.PreCommitSector, - Params: enc, - Value: types.NewInt(0), // TODO: need to ensure sufficient collateral - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return errors.Wrap(err, "pushing message to mpool") - } - - go func() { - // TODO: maybe just mark this down in the datastore and handle it differently? This feels complicated to restart - mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return - } - - randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - - err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { - go func() { - rand, err := m.api.ChainGetRandomness(ctx, ts, nil, ts.Height()-randHeight) - if err != nil { - log.Error(errors.Errorf("failed to get randomness for computing seal proof: %w", err)) - return - } - - // TODO: should this get scheduled to preserve proper resource consumption? - proof, err := m.secst.SealComputeProof(ctx, sinfo.SectorID, rand) - if err != nil { - log.Error(errors.Errorf("computing seal proof failed: %w", err)) - return - } - - params := &actors.SectorProveCommitInfo{ - Proof: proof, - SectorID: sinfo.SectorID, - //DealIDs: deals, - } - - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - log.Errorf(errors.Wrap(aerr, "could not serialize commit sector parameters")) - return - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.ProveCommitSector, - Params: enc, - Value: types.NewInt(0), // TODO: need to ensure sufficient collateral - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return errors.Wrap(err, "pushing message to mpool") - } - - // TODO: now wait for this to get included and handle errors? - _, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - log.Errorf("failed to wait for porep inclusion: %s", err) - return - } - - m.beginPosting(ctx) - }() - - return nil - }, func(ts *types.TipSet) error { - log.Warn("revert in interactive commit sector step") - return nil - }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) - if err != nil { - return - } - }() - - return nil + return m.SealSector(ctx, sinfo.SectorID) } func (m *Miner) runPreflightChecks(ctx context.Context) error { diff --git a/storage/sector/store.go b/storage/sector/store.go index 19460e8bb..3e498f052 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -31,7 +31,7 @@ type dealMapping struct { Committed bool } -type TicketFn func(context.Context) (*sectorbuilder.SealSeed, error) +type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) // TODO: eventually handle sector storage here instead of in rust-sectorbuilder type Store struct { @@ -60,10 +60,6 @@ func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn Ticke } } -func (s *Store) Service() { - go s.service() -} - func (s *Store) restartSealing() { sectors, err := s.sb.GetAllStagedSectors() if err != nil { @@ -92,6 +88,15 @@ func (s *Store) restartSealing() { } } +func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, error) { + status, err := s.sb.SealStatus(sid) + if err != nil { + return nil, err + } + + return &status, nil +} + func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { sectorID, err = s.sb.AddPiece(ref, size, r) if err != nil { @@ -190,35 +195,6 @@ func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, rand []by panic("TODO") } -func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) { - s.waitingLk.Lock() - var at = -1 - for i, ch := range s.incoming { - if ch == c { - at = i - } - } - if at == -1 { - s.waitingLk.Unlock() - return - } - if len(s.incoming) > 1 { - last := len(s.incoming) - 1 - s.incoming[at] = s.incoming[last] - s.incoming[last] = nil - } - s.incoming = s.incoming[:len(s.incoming)-1] - s.waitingLk.Unlock() -} - -func (s *Store) Incoming() <-chan sectorbuilder.SectorSealingStatus { - ch := make(chan sectorbuilder.SectorSealingStatus, 8) - s.waitingLk.Lock() - s.incoming = append(s.incoming, ch) - s.waitingLk.Unlock() - return ch -} - func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) { s.waitingLk.Lock() watch, ok := s.waiting[sector] From b9f8addd21f2fb604c93c9dbd2321f7023c6817f Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 31 Oct 2019 10:24:59 -0700 Subject: [PATCH 097/230] Invert faucet rate limiting Rate limits needs to be ordered from specific to generic to ensure that a single user will not exhaust the generic limiter before being limited by the more specific. Resolves #517 --- cmd/lotus-fountain/main.go | 57 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 6c27758a6..6d590bb0c 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -136,19 +136,6 @@ type handler struct { } func (h *handler) send(w http.ResponseWriter, r *http.Request) { - // General limiter to allow throttling all messages that can make it into the mpool - if !h.limiter.Allow() { - http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) - return - } - - // Limit based on IP - limiter := h.limiter.GetIPLimiter(r.RemoteAddr) - if !limiter.Allow() { - http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) - return - } - to, err := address.NewFromString(r.FormValue("address")) if err != nil { w.WriteHeader(400) @@ -157,12 +144,25 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { } // Limit based on wallet address - limiter = h.limiter.GetWalletLimiter(to.String()) + limiter := h.limiter.GetWalletLimiter(to.String()) if !limiter.Allow() { http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) return } + // Limit based on IP + limiter = h.limiter.GetIPLimiter(r.RemoteAddr) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + + // General limiter to allow throttling all messages that can make it into the mpool + if !h.limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ Value: sendPerRequest, From: h.from, @@ -181,19 +181,6 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { } func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { - // General limiter owner allow throttling all messages that can make it into the mpool - if !h.colLimiter.Allow() { - http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) - return - } - - // Limit based on IP - limiter := h.colLimiter.GetIPLimiter(r.RemoteAddr) - if !limiter.Allow() { - http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) - return - } - owner, err := address.NewFromString(r.FormValue("address")) if err != nil { w.WriteHeader(400) @@ -215,11 +202,25 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { log.Infof("mkactor on %s", owner) // Limit based on wallet address - limiter = h.colLimiter.GetWalletLimiter(owner.String()) + limiter := h.colLimiter.GetWalletLimiter(owner.String()) if !limiter.Allow() { http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) return } + + // Limit based on IP + limiter = h.colLimiter.GetIPLimiter(r.RemoteAddr) + if !limiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + + // General limiter owner allow throttling all messages that can make it into the mpool + if !h.colLimiter.Allow() { + http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) + return + } + collateral, err := h.api.StatePledgeCollateral(r.Context(), nil) if err != nil { w.WriteHeader(400) From 34998d936281ca6ecf0ac8fe9f7fc19fb5bbf69f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 31 Oct 2019 11:46:53 -0700 Subject: [PATCH 098/230] add file i forgot --- storage/sealing.go | 141 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 storage/sealing.go diff --git a/storage/sealing.go b/storage/sealing.go new file mode 100644 index 000000000..eeeca0f64 --- /dev/null +++ b/storage/sealing.go @@ -0,0 +1,141 @@ +package storage + +import ( + "context" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + "github.com/pkg/errors" + "golang.org/x/xerrors" +) + +func (m *Miner) SealSector(ctx context.Context, sid uint64) error { + log.Info("committing sector") + + ssize, err := m.SectorSize(ctx) + if err != nil { + return xerrors.Errorf("failed to check out own sector size: %w", err) + } + + _ = ssize + + sinfo, err := m.secst.SectorStatus(sid) + if err != nil { + return xerrors.Errorf("failed to check status for sector %d: %w", sid, err) + } + + params := &actors.SectorPreCommitInfo{ + CommD: sinfo.CommD[:], + CommR: sinfo.CommR[:], + Epoch: sinfo.Ticket.BlockHeight, + + //DealIDs: deals, + SectorNumber: sinfo.SectorID, + } + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return errors.Wrap(aerr, "could not serialize commit sector parameters") + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.PreCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return errors.Wrap(err, "pushing message to mpool") + } + + go m.waitForPreCommitMessage(context.TODO(), sinfo.SectorID, smsg.Cid()) + + // TODO: maybe return a wait channel? + return nil +} + +func (m *Miner) waitForPreCommitMessage(ctx context.Context, sid uint64, mcid cid.Cid) { + // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts + mw, err := m.api.StateWaitMsg(ctx, mcid) + if err != nil { + return + } + + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay + + err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { + return m.scheduleComputeProof(ctx, sid, ts, randHeight) + }, func(ts *types.TipSet) error { + log.Warn("revert in interactive commit sector step") + return nil + }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) + if err != nil { + log.Warn("waitForPreCommitMessage ChainAt errored: ", err) + } +} + +func (m *Miner) scheduleComputeProof(ctx context.Context, sid uint64, ts *types.TipSet, rheight uint64) error { + go func() { + rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-rheight)) + if err != nil { + log.Error(errors.Errorf("failed to get randomness for computing seal proof: %w", err)) + return + } + + proof, err := m.secst.SealComputeProof(ctx, sid, rand) + if err != nil { + log.Error(errors.Errorf("computing seal proof failed: %w", err)) + return + } + + params := &actors.SectorProveCommitInfo{ + Proof: proof, + SectorID: sid, + //DealIDs: deals, + } + + _ = params + enc, aerr := actors.SerializeParams(nil) + if aerr != nil { + log.Error(errors.Wrap(aerr, "could not serialize commit sector parameters")) + return + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.ProveCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + log.Error(errors.Wrap(err, "pushing message to mpool")) + } + + // TODO: now wait for this to get included and handle errors? + mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + log.Errorf("failed to wait for porep inclusion: %s", err) + return + } + + if mw.Receipt.ExitCode != 0 { + log.Error("UNHANDLED: submitting sector proof failed") + return + } + + m.beginPosting(ctx) + }() + + return nil +} From ba937cf859dea578fcab3c97d42117e5de324064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 31 Oct 2019 20:03:26 +0100 Subject: [PATCH 099/230] Fix sectorbuilder.VerifySeal --- chain/deals/provider.go | 6 +++--- lib/sectorbuilder/sectorbuilder.go | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 132e3bf12..bf71e3460 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -42,7 +42,7 @@ type Provider struct { askLk sync.Mutex sminer *storage.Miner - full api.FullNode + full api.FullNode // TODO: Use a custom protocol or graphsync in the future // TODO: GC @@ -80,8 +80,8 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.Staging h := &Provider{ sminer: sminer, - dag: dag, - full: fullNode, + dag: dag, + full: fullNode, pricePerByteBlock: types.NewInt(3), // TODO: allow setting minPieceSize: 1, diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 0c1aeb66e..f17dd4121 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -142,7 +142,10 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address copy(seeda[:], seed) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, nil) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, []sectorbuilder.PublicPieceInfo{{ + Size: UserBytesForSectorSize(sectorSize), // TODO: Provide the real piece size? + CommP: commDa, + }}) } func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo { From 3cde267a2a28d8150a6cdd1330222a5194a83d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 31 Oct 2019 22:01:44 +0100 Subject: [PATCH 100/230] Move statestore to lib --- chain/deals/client.go | 3 +- chain/deals/provider.go | 3 +- chain/deals/state_store.go | 80 +++----------------------------- lib/statestore/store.go | 95 ++++++++++++++++++++++++++++++++++++++ node/impl/storminer.go | 2 +- 5 files changed, 107 insertions(+), 76 deletions(-) create mode 100644 lib/statestore/store.go diff --git a/chain/deals/client.go b/chain/deals/client.go index fa70dbe99..3ea36c2e7 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/impl/full" "github.com/ipfs/go-cid" @@ -71,7 +72,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * discovery: discovery, mpool: mpool, - deals: ClientStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, + deals: ClientStateStore{statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))}, conns: map[cid.Cid]inet.Stream{}, incoming: make(chan *ClientDeal, 16), diff --git a/chain/deals/provider.go b/chain/deals/provider.go index bf71e3460..aa47a45c5 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/lib/statestore" "sync" cid "github.com/ipfs/go-cid" @@ -95,7 +96,7 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.Staging actor: minerAddress, - deals: MinerStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}}, + deals: MinerStateStore{statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))}, ds: ds, } diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go index cde6240be..b654816c6 100644 --- a/chain/deals/state_store.go +++ b/chain/deals/state_store.go @@ -2,77 +2,18 @@ package deals import ( "bytes" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - "golang.org/x/xerrors" ) -type StateStore struct { - ds datastore.Datastore -} - -func (st *StateStore) Begin(i cid.Cid, state interface{}) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if has { - return xerrors.Errorf("Already tracking state for %s", i) - } - - b, err := cborrpc.Dump(state) - if err != nil { - return err - } - - return st.ds.Put(k, b) -} - -func (st *StateStore) End(i cid.Cid) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - return st.ds.Delete(k) -} - -func (st *StateStore) mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { - k := datastore.NewKey(i.String()) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - - cur, err := st.ds.Get(k) - if err != nil { - return err - } - - mutated, err := mutator(cur) - if err != nil { - return err - } - - return st.ds.Put(k, mutated) -} - type MinerStateStore struct { - StateStore + *statestore.StateStore } func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error) error { - return st.mutate(i, minerMutator(mutator)) + return st.Mutate(i, minerMutator(mutator)) } func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { @@ -92,11 +33,11 @@ func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { } type ClientStateStore struct { - StateStore + *statestore.StateStore } func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) error) error { - return st.mutate(i, clientMutator(mutator)) + return st.Mutate(i, clientMutator(mutator)) } func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { @@ -118,18 +59,11 @@ func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { func (st *ClientStateStore) ListClient() ([]ClientDeal, error) { var out []ClientDeal - res, err := st.ds.Query(query.Query{}) + l, err := st.List() if err != nil { return nil, err } - defer res.Close() - - for { - res, ok := res.NextSync() - if !ok { - break - } - + for _, res := range l { var deal ClientDeal err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), &deal) if err != nil { diff --git a/lib/statestore/store.go b/lib/statestore/store.go new file mode 100644 index 000000000..50c5eeb61 --- /dev/null +++ b/lib/statestore/store.go @@ -0,0 +1,95 @@ +package statestore + +import ( + "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + "golang.org/x/xerrors" +) + +type StateStore struct { + ds datastore.Datastore +} + +func New(ds datastore.Datastore) *StateStore { + return &StateStore{ds:ds} +} + +func (st *StateStore) Begin(i cid.Cid, state interface{}) error { + k := datastore.NewKey(i.String()) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if has { + return xerrors.Errorf("Already tracking state for %s", i) + } + + b, err := cborrpc.Dump(state) + if err != nil { + return err + } + + return st.ds.Put(k, b) +} + +func (st *StateStore) End(i cid.Cid) error { + k := datastore.NewKey(i.String()) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if !has { + return xerrors.Errorf("No state for %s", i) + } + return st.ds.Delete(k) +} + +func (st *StateStore) Mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { + k := datastore.NewKey(i.String()) + has, err := st.ds.Has(k) + if err != nil { + return err + } + if !has { + return xerrors.Errorf("No state for %s", i) + } + + cur, err := st.ds.Get(k) + if err != nil { + return err + } + + mutated, err := mutator(cur) + if err != nil { + return err + } + + return st.ds.Put(k, mutated) +} + +func (st *StateStore) List() ([]query.Entry, error) { + var out []query.Entry + + res, err := st.ds.Query(query.Query{}) + if err != nil { + return nil, err + } + defer res.Close() + + for { + res, ok := res.NextSync() + if !ok { + break + } + if res.Error != nil { + return nil, res.Error + } + + out = append(out, res.Entry) + } + + return out, nil +} + diff --git a/node/impl/storminer.go b/node/impl/storminer.go index c763dab5e..b8b101ec3 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -47,7 +47,7 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { return } - if err := sm.Sectors.SealSector(ctx, sectorId); err != nil { + if err := sm.Miner.SealSector(ctx, sectorId); err != nil { log.Error(err) return } From a954ebc6b6177910cf99ca6126856aed8b856e2a Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 31 Oct 2019 15:04:13 -0700 Subject: [PATCH 101/230] Add command to unregister miner from full node manually --- cli/cmd.go | 1 + cli/miner.go | 32 ++++++++++++++++++++++++++++++++ go.mod | 1 - go.sum | 5 ++--- node/modules/storageminer.go | 6 +++++- 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 cli/miner.go diff --git a/cli/cmd.go b/cli/cmd.go index f80c80453..30e9ba01a 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -122,6 +122,7 @@ var Commands = []*cli.Command{ sendCmd, stateCmd, syncCmd, + unregisterMinerCmd, versionCmd, walletCmd, } diff --git a/cli/miner.go b/cli/miner.go new file mode 100644 index 000000000..074faf962 --- /dev/null +++ b/cli/miner.go @@ -0,0 +1,32 @@ +package cli + +import ( + "fmt" + + "github.com/filecoin-project/lotus/chain/address" + "gopkg.in/urfave/cli.v2" +) + +var unregisterMinerCmd = &cli.Command{ + Name: "unregister-miner", + Usage: "Manually unregister miner actor", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + if !cctx.Args().Present() { + return fmt.Errorf("must pass address of miner to unregister") + } + + maddr, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return err + } + + return api.MinerUnregister(ctx, maddr) + }, +} diff --git a/go.mod b/go.mod index b8eec445f..67f47edc4 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,6 @@ require ( go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect - golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect golang.org/x/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 google.golang.org/api v0.9.0 // indirect diff --git a/go.sum b/go.sum index 3d9bc15a9..fd9e20aa9 100644 --- a/go.sum +++ b/go.sum @@ -613,9 +613,9 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= +github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= @@ -746,9 +746,8 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4f9df1c42..49a8d1dd5 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "fmt" "path/filepath" "github.com/ipfs/go-bitswap" @@ -154,7 +155,10 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { log.Infof("Registering miner '%s' with full node", minerAddr) - return api.MinerRegister(ctx, minerAddr) + if err := api.MinerRegister(ctx, minerAddr); err != nil { + return fmt.Errorf("Failed to register miner: %s\nIf you are certain no other storage miner instance is running, try running 'lotus unregister-miner %s' and restarting the storage miner", err, minerAddr) + } + return nil }, OnStop: func(ctx context.Context) error { log.Infof("Unregistering miner '%s' from full node", minerAddr) From 9a5adfed3ae66cc02f3f3c2892cdfbae8f3a321a Mon Sep 17 00:00:00 2001 From: Steven Li Date: Fri, 1 Nov 2019 08:56:15 +0800 Subject: [PATCH 102/230] change with @Kubuxu's comment --- chain/types/blockheader.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 1a1d5b875..204208e0f 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -170,12 +170,13 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { max(h) == 2^256-1 which in terms of integer math means: (h(vrfout) + 1) * totalPower <= e * minerPower * 2^256 + in 2^256 space, it is equivalent to: + h(vrfout) * totalPower < e * minerPower * 2^256 */ h := sha256.Sum256(eproof) lhs := BigFromBytes(h[:]).Int - lhs = lhs.Add(lhs, NewInt(1).Int) lhs = lhs.Mul(lhs, totpow.Int) // rhs = minerPower * 2^256 @@ -183,8 +184,8 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { rhs := new(big.Int).Lsh(mpow.Int, 256) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) - // return true if lhs is less than or equal to rhs - return lhs.Cmp(rhs) <= 0 + // h(vrfout) * totalPower < e * minerPower * 2^256? + return lhs.Cmp(rhs) == -1 } func (t *Ticket) Equals(ot *Ticket) bool { From dd8642c43cf8488a86117f02d56d833d0b67abf3 Mon Sep 17 00:00:00 2001 From: steven004 Date: Fri, 1 Nov 2019 09:43:22 +0800 Subject: [PATCH 103/230] Omit double delaration --- chain/store/weight.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/chain/store/weight.go b/chain/store/weight.go index 3f52112cb..999166f89 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -11,8 +11,6 @@ import ( ) var zero = types.NewInt(0) -var wRatio_num = build.WRatioNum -var wRatio_den = build.WRatioDen func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigInt, error) { if ts == nil { @@ -41,15 +39,15 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn log2P = int64(tpow.BitLen() - 1) } else { // Not really expect to be here ... - return types.EmptyInt, xerrors.Errorf("All power in the net is gone. The net is dead!") + return types.EmptyInt, xerrors.Errorf("All power in the net is gone. You network might be disconnected, or the net is dead!") } out.Add(out, big.NewInt(log2P << 8)) // (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den) - eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * wRatio_num) << 8) - eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch * wRatio_den))) + eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * build.WRatioNum) << 8) + eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch * build.WRatioDen))) out.Add(out, eWeight) return types.BigInt{Int: out}, nil From fc9091cc89eddc3ac586651805950f7cb82f3f28 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 31 Oct 2019 20:57:10 -0700 Subject: [PATCH 104/230] Get interactive porep sector sealing mostly working --- chain/actors/actor_storagemarket.go | 18 ++++++++++--- cmd/lotus-storage-miner/info.go | 40 +++++++++-------------------- cmd/lotus/daemon.go | 3 ++- extern/go-sectorbuilder | 2 +- go.sum | 1 + lib/sectorbuilder/sectorbuilder.go | 9 ++++--- node/builder.go | 1 - node/modules/services.go | 14 ---------- node/options.go | 3 ++- peermgr/peermgr.go | 4 ++- storage/sealing.go | 22 +++++++++++++--- storage/sector/store.go | 14 ++++++++-- 12 files changed, 70 insertions(+), 61 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 6eeeb7f99..cc2e7a96e 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type StorageMarketActor struct{} @@ -605,6 +606,7 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type return nil, aerrors.HandleExternalError(err, "loading deals amt") } + var pieces []sectorbuilder.PublicPieceInfo for _, deal := range params.DealIDs { var dealInfo OnChainDeal if err := deals.Get(deal, &dealInfo); err != nil { @@ -614,13 +616,21 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - _ = dealInfo + var commP [32]byte + copy(commP[:], dealInfo.Deal.Proposal.PieceRef) + + pieces = append(pieces, sectorbuilder.PublicPieceInfo{ + Size: dealInfo.Deal.Proposal.PieceSize, + CommP: commP, + }) } - // TODO: rust-fil-proofs-magic - var commDoutput [32]byte + commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces) + if err != nil { + return nil, aerrors.Absorb(err, 4, "failed to generate data commitment from pieces") + } - return commDoutput[:], nil + return commd[:], nil } /* diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 436fc4686..213204a99 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -6,7 +6,6 @@ import ( "gopkg.in/urfave/cli.v2" - sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -50,10 +49,13 @@ var infoCmd = &cli.Command{ return err } - fmt.Println("Sealed Sectors:\t", sinfo.SealedCount) - fmt.Println("Sealing Sectors:\t", sinfo.SealingCount) - fmt.Println("Pending Sectors:\t", sinfo.PendingCount) - fmt.Println("Failed Sectors:\t", sinfo.FailedCount) + /* + fmt.Println("Sealed Sectors:\t", sinfo.SealedCount) + fmt.Println("Sealing Sectors:\t", sinfo.SealingCount) + fmt.Println("Pending Sectors:\t", sinfo.PendingCount) + fmt.Println("Failed Sectors:\t", sinfo.FailedCount) + */ + fmt.Println(sinfo) // TODO: grab actr state / info // * Sector size @@ -63,22 +65,14 @@ var infoCmd = &cli.Command{ }, } -type SectorsInfo struct { - TotalCount int - SealingCount int - FailedCount int - SealedCount int - PendingCount int -} - -func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, error) { +func sectorsInfo(ctx context.Context, napi api.StorageMiner) (map[string]int, error) { sectors, err := napi.SectorsList(ctx) if err != nil { return nil, err } - out := SectorsInfo{ - TotalCount: len(sectors), + out := map[string]int{ + "Total": len(sectors), } for _, s := range sectors { st, err := napi.SectorsStatus(ctx, s) @@ -86,18 +80,8 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, erro return nil, err } - switch st.State { - case sectorstate.Sealed: - out.SealedCount++ - case sectorstate.Pending: - out.PendingCount++ - case sectorstate.Sealing: - out.SealingCount++ - case sectorstate.Failed: - out.FailedCount++ - case sectorstate.Unknown: - } + out[st.State.String()]++ } - return &out, nil + return out, nil } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c9b736c72..62b579eb1 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -4,9 +4,10 @@ package main import ( "context" - "github.com/filecoin-project/lotus/peermgr" "io/ioutil" + "github.com/filecoin-project/lotus/peermgr" + "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index e8231db9d..06775780d 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit e8231db9dc64e79f7049e25c538b77b88eef54dc +Subproject commit 06775780d8e4db7ba123a7c76073e10d7785b076 diff --git a/go.sum b/go.sum index 76f9b1a65..09d5a1111 100644 --- a/go.sum +++ b/go.sum @@ -749,6 +749,7 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index f17dd4121..bd57a7bda 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -142,10 +142,7 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address copy(seeda[:], seed) proverIDa := addressToProverID(proverID) - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof, []sectorbuilder.PublicPieceInfo{{ - Size: UserBytesForSectorSize(sectorSize), // TODO: Provide the real piece size? - CommP: commDa, - }}) + return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof) } func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo { @@ -170,6 +167,10 @@ func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen] return commP, werr() } +func GenerateDataCommitment(ssize uint64, pieces []PublicPieceInfo) ([CommLen]byte, error) { + return sectorbuilder.GenerateDataCommitment(ssize, pieces) +} + func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { f, ok := r.(*os.File) if ok { diff --git a/node/builder.go b/node/builder.go index 26b6bbd7f..ded10f3d6 100644 --- a/node/builder.go +++ b/node/builder.go @@ -242,7 +242,6 @@ func Online() Option { Override(new(*deals.Provider), deals.NewProvider), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleDealsKey, modules.HandleDeals), - Override(RunSectorServiceKey, modules.RunSectorService), Override(RegisterMinerKey, modules.RegisterMiner), ), ) diff --git a/node/modules/services.go b/node/modules/services.go index 3c38f4461..c5c3df125 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -15,7 +15,6 @@ import ( "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/retrieval/discovery" - "github.com/filecoin-project/lotus/storage/sector" ) func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { @@ -79,19 +78,6 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) { }) } -func RunSectorService(lc fx.Lifecycle, secst *sector.Store) { - lc.Append(fx.Hook{ - OnStart: func(context.Context) error { - secst.Service() - return nil - }, - OnStop: func(context.Context) error { - secst.Stop() - return nil - }, - }) -} - func RetrievalResolver(l *discovery.Local) discovery.PeerResolver { return discovery.Multi(l) } diff --git a/node/options.go b/node/options.go index 58576a950..9c8a79710 100644 --- a/node/options.go +++ b/node/options.go @@ -1,8 +1,9 @@ package node import ( - "go.uber.org/fx" "reflect" + + "go.uber.org/fx" ) // Option is a functional option which can be used with the New function to diff --git a/peermgr/peermgr.go b/peermgr/peermgr.go index d8a0bea77..3c4053da1 100644 --- a/peermgr/peermgr.go +++ b/peermgr/peermgr.go @@ -2,10 +2,11 @@ package peermgr import ( "context" - "github.com/filecoin-project/lotus/node/modules/dtypes" "sync" "time" + "github.com/filecoin-project/lotus/node/modules/dtypes" + host "github.com/libp2p/go-libp2p-core/host" net "github.com/libp2p/go-libp2p-core/network" peer "github.com/libp2p/go-libp2p-core/peer" @@ -43,6 +44,7 @@ type PeerMgr struct { } func NewPeerMgr(h host.Host, dht *dht.IpfsDHT, bootstrap dtypes.BootstrapPeers) *PeerMgr { + bootstrap = nil pm := &PeerMgr{ h: h, dht: dht, diff --git a/storage/sealing.go b/storage/sealing.go index eeeca0f64..2a3d02d7f 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,6 +2,7 @@ package storage import ( "context" + "fmt" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -12,7 +13,7 @@ import ( ) func (m *Miner) SealSector(ctx context.Context, sid uint64) error { - log.Info("committing sector") + log.Info("committing sector: ", sid) ssize, err := m.SectorSize(ctx) if err != nil { @@ -21,6 +22,11 @@ func (m *Miner) SealSector(ctx context.Context, sid uint64) error { _ = ssize + log.Info("performing sector replication...") + if err := m.secst.SealPreCommit(ctx, sid); err != nil { + return xerrors.Errorf("seal pre commit failed: %w", err) + } + sinfo, err := m.secst.SectorStatus(sid) if err != nil { return xerrors.Errorf("failed to check status for sector %d: %w", sid, err) @@ -49,6 +55,7 @@ func (m *Miner) SealSector(ctx context.Context, sid uint64) error { GasPrice: types.NewInt(1), } + log.Info("submitting precommit for sector: ", sid) smsg, err := m.api.MpoolPushMessage(ctx, msg) if err != nil { return errors.Wrap(err, "pushing message to mpool") @@ -67,7 +74,13 @@ func (m *Miner) waitForPreCommitMessage(ctx context.Context, sid uint64, mcid ci return } + if mw.Receipt.ExitCode != 0 { + log.Error("sector precommit failed: ", mw.Receipt.ExitCode) + return + } + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay + log.Infof("precommit for sector %d made it on chain, will start post computation at height %d", sid, randHeight) err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { return m.scheduleComputeProof(ctx, sid, ts, randHeight) @@ -81,16 +94,17 @@ func (m *Miner) waitForPreCommitMessage(ctx context.Context, sid uint64, mcid ci } func (m *Miner) scheduleComputeProof(ctx context.Context, sid uint64, ts *types.TipSet, rheight uint64) error { + log.Info("scheduling post computation...") go func() { rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-rheight)) if err != nil { - log.Error(errors.Errorf("failed to get randomness for computing seal proof: %w", err)) + log.Error(fmt.Errorf("failed to get randomness for computing seal proof: %w", err)) return } - proof, err := m.secst.SealComputeProof(ctx, sid, rand) + proof, err := m.secst.SealComputeProof(ctx, sid, rheight, rand) if err != nil { - log.Error(errors.Errorf("computing seal proof failed: %w", err)) + log.Error(fmt.Errorf("computing seal proof failed: %w", err)) return } diff --git a/storage/sector/store.go b/storage/sector/store.go index 3e498f052..414bf91ec 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -191,8 +191,18 @@ func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) error { return nil } -func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, rand []byte) ([]byte, error) { - panic("TODO") +func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height uint64, rand []byte) ([]byte, error) { + var tick [32]byte + copy(tick[:], rand) + + sco, err := s.sb.SealCommit(sectorID, sectorbuilder.SealSeed{ + BlockHeight: height, + TicketBytes: tick, + }) + if err != nil { + return nil, err + } + return sco.Proof, nil } func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) { From 1583cf25938466082a4459dfa32ab3d901751d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 11:05:48 +0100 Subject: [PATCH 105/230] Strip unused functionality from sectorstore --- lib/statestore/store.go | 3 +- storage/sector/store.go | 70 ++--------------------------------------- 2 files changed, 4 insertions(+), 69 deletions(-) diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 50c5eeb61..bf884788a 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -13,7 +13,7 @@ type StateStore struct { } func New(ds datastore.Datastore) *StateStore { - return &StateStore{ds:ds} + return &StateStore{ds: ds} } func (st *StateStore) Begin(i cid.Cid, state interface{}) error { @@ -92,4 +92,3 @@ func (st *StateStore) List() ([]query.Entry, error) { return out, nil } - diff --git a/storage/sector/store.go b/storage/sector/store.go index 414bf91ec..855fe2c39 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -35,56 +35,18 @@ type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) // TODO: eventually handle sector storage here instead of in rust-sectorbuilder type Store struct { - waitingLk sync.Mutex - sb *sectorbuilder.SectorBuilder tktFn TicketFn dealsLk sync.Mutex deals datastore.Datastore - - waiting map[uint64]chan struct{} - incoming []chan sectorbuilder.SectorSealingStatus - // TODO: outdated chan - - closeCh chan struct{} } func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store { return &Store{ - sb: sb, - tktFn: tktFn, - deals: namespace.Wrap(ds, sectorDealsPrefix), - waiting: map[uint64]chan struct{}{}, - closeCh: make(chan struct{}), - } -} - -func (s *Store) restartSealing() { - sectors, err := s.sb.GetAllStagedSectors() - if err != nil { - return - } - - for _, sid := range sectors { - status, err := s.sb.SealStatus(sid) - if err != nil { - return - } - - if status.State != sealing_state.CommittingPaused { // TODO: Also handle PreCommit! - continue - } - - log.Infof("Sector %d is in paused state, resuming sealing", sid) - go func() { - // TODO: when we refactor wait-for-seal below, care about this output too - // (see SealSector below) - _, err := s.sb.ResumeSealCommit(sid) - if err != nil { - return - } - }() + sb: sb, + tktFn: tktFn, + deals: namespace.Wrap(ds, sectorDealsPrefix), } } @@ -103,13 +65,6 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 return 0, err } - s.waitingLk.Lock() - _, exists := s.waiting[sectorID] - if !exists { // pieces can share sectors - s.waiting[sectorID] = make(chan struct{}) - } - s.waitingLk.Unlock() - s.dealsLk.Lock() defer s.dealsLk.Unlock() @@ -205,21 +160,6 @@ func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height ui return sco.Proof, nil } -func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) { - s.waitingLk.Lock() - watch, ok := s.waiting[sector] - s.waitingLk.Unlock() - if ok { - select { - case <-watch: - case <-ctx.Done(): - return sectorbuilder.SectorSealingStatus{}, ctx.Err() - } - } - - return s.sb.SealStatus(sector) -} - func (s *Store) Commited() ([]sectorbuilder.SectorSealingStatus, error) { l, err := s.sb.GetAllStagedSectors() if err != nil { @@ -265,7 +205,3 @@ func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte return s.sb.GeneratePoSt(ssi, seed, faults) } - -func (s *Store) Stop() { - close(s.closeCh) -} From 2343ebc5b81fae8b3c66f8143d7e0fa2bbff3347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 12:07:05 +0100 Subject: [PATCH 106/230] statestore: Use reflect for mutators --- chain/deals/client.go | 12 ++++-- chain/deals/provider.go | 6 +-- chain/deals/state_store.go | 77 -------------------------------------- lib/statestore/store.go | 52 +++++++++++++++++++++---- 4 files changed, 55 insertions(+), 92 deletions(-) delete mode 100644 chain/deals/state_store.go diff --git a/chain/deals/client.go b/chain/deals/client.go index 3ea36c2e7..e89a9886f 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -46,7 +46,7 @@ type Client struct { discovery *discovery.Local mpool full.MpoolAPI - deals ClientStateStore + deals *statestore.StateStore conns map[cid.Cid]inet.Stream incoming chan *ClientDeal @@ -72,7 +72,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * discovery: discovery, mpool: mpool, - deals: ClientStateStore{statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))}, + deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), conns: map[cid.Cid]inet.Stream{}, incoming: make(chan *ClientDeal, 16), @@ -130,7 +130,7 @@ func (c *Client) onIncoming(deal *ClientDeal) { func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { log.Infof("Deal %s updated state to %d", update.id, update.newState) var deal ClientDeal - err := c.deals.MutateClient(update.id, func(d *ClientDeal) error { + err := c.deals.Mutate(update.id, func(d *ClientDeal) error { d.State = update.newState deal = *d return nil @@ -286,7 +286,11 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t } func (c *Client) List() ([]ClientDeal, error) { - return c.deals.ListClient() + var out []ClientDeal + if err := c.deals.List(&out); err != nil { + return nil, err + } + return out, nil } func (c *Client) Stop() { diff --git a/chain/deals/provider.go b/chain/deals/provider.go index aa47a45c5..02966c05d 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -49,7 +49,7 @@ type Provider struct { // TODO: GC dag dtypes.StagingDAG - deals MinerStateStore + deals *statestore.StateStore ds dtypes.MetadataDS conns map[cid.Cid]inet.Stream @@ -96,7 +96,7 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.Staging actor: minerAddress, - deals: MinerStateStore{statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))}, + deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), ds: ds, } @@ -164,7 +164,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { return } var deal MinerDeal - err := p.deals.MutateMiner(update.id, func(d *MinerDeal) error { + err := p.deals.Mutate(update.id, func(d *MinerDeal) error { d.State = update.newState if update.mut != nil { update.mut(d) diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go deleted file mode 100644 index b654816c6..000000000 --- a/chain/deals/state_store.go +++ /dev/null @@ -1,77 +0,0 @@ -package deals - -import ( - "bytes" - "github.com/filecoin-project/lotus/lib/statestore" - - "github.com/filecoin-project/lotus/lib/cborrpc" - "github.com/ipfs/go-cid" -) - -type MinerStateStore struct { - *statestore.StateStore -} - -func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error) error { - return st.Mutate(i, minerMutator(mutator)) -} - -func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) { - return func(in []byte) ([]byte, error) { - deal := new(MinerDeal) - err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) - if err != nil { - return nil, err - } - - if err := m(deal); err != nil { - return nil, err - } - - return cborrpc.Dump(deal) - } -} - -type ClientStateStore struct { - *statestore.StateStore -} - -func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) error) error { - return st.Mutate(i, clientMutator(mutator)) -} - -func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) { - return func(in []byte) ([]byte, error) { - deal := new(ClientDeal) - err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal) - if err != nil { - return nil, err - } - - if err := m(deal); err != nil { - return nil, err - } - - return cborrpc.Dump(deal) - } -} - -func (st *ClientStateStore) ListClient() ([]ClientDeal, error) { - var out []ClientDeal - - l, err := st.List() - if err != nil { - return nil, err - } - for _, res := range l { - var deal ClientDeal - err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), &deal) - if err != nil { - return nil, err - } - - out = append(out, deal) - } - - return out, nil -} diff --git a/lib/statestore/store.go b/lib/statestore/store.go index bf884788a..10a4de9bc 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -1,11 +1,13 @@ package statestore import ( + "bytes" "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "golang.org/x/xerrors" + "reflect" ) type StateStore struct { @@ -46,7 +48,33 @@ func (st *StateStore) End(i cid.Cid) error { return st.ds.Delete(k) } -func (st *StateStore) Mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { +func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { + rmut := reflect.ValueOf(mutator) + + return func(in []byte) ([]byte, error) { + state := reflect.New(rmut.Type().In(0).Elem()) + + err := cborrpc.ReadCborRPC(bytes.NewReader(in), state.Interface()) + if err != nil { + return nil, err + } + + out := rmut.Call([]reflect.Value{state}) + + if err := out[0].Interface().(error); err != nil { + return nil, err + } + + return cborrpc.Dump(state.Interface()) + } +} + +// mutator func(*T) error +func (st *StateStore) Mutate(i cid.Cid, mutator interface{}) error { + return st.mutate(i, cborMutator(mutator)) +} + +func (st *StateStore) mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { k := datastore.NewKey(i.String()) has, err := st.ds.Has(k) if err != nil { @@ -69,26 +97,34 @@ func (st *StateStore) Mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) er return st.ds.Put(k, mutated) } -func (st *StateStore) List() ([]query.Entry, error) { - var out []query.Entry - +// out: *[]T +func (st *StateStore) List(out interface{}) error { res, err := st.ds.Query(query.Query{}) if err != nil { - return nil, err + return err } defer res.Close() + outT := reflect.TypeOf(out).Elem().Elem() + rout := reflect.ValueOf(out) + for { res, ok := res.NextSync() if !ok { break } if res.Error != nil { - return nil, res.Error + return res.Error } - out = append(out, res.Entry) + elem := reflect.New(outT) + err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) + if err != nil { + return err + } + + rout.Set(reflect.Append(rout.Elem(), elem.Elem())) } - return out, nil + return nil } From 02c3be9099444669ddf781a3276b40475328ef72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 12:14:32 +0100 Subject: [PATCH 107/230] statestore: More generic keys --- lib/statestore/store.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 10a4de9bc..2fd4a4384 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -2,8 +2,8 @@ package statestore import ( "bytes" + "fmt" "github.com/filecoin-project/lotus/lib/cborrpc" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "golang.org/x/xerrors" @@ -18,8 +18,19 @@ func New(ds datastore.Datastore) *StateStore { return &StateStore{ds: ds} } -func (st *StateStore) Begin(i cid.Cid, state interface{}) error { - k := datastore.NewKey(i.String()) +func toKey(k interface{}) datastore.Key { + switch t := k.(type) { + case uint64: + return datastore.NewKey(fmt.Sprint(t)) + case fmt.Stringer: + return datastore.NewKey(t.String()) + default: + panic("unexpected key type") + } +} + +func (st *StateStore) Begin(i interface{}, state interface{}) error { + k := toKey(i) has, err := st.ds.Has(k) if err != nil { return err @@ -36,8 +47,8 @@ func (st *StateStore) Begin(i cid.Cid, state interface{}) error { return st.ds.Put(k, b) } -func (st *StateStore) End(i cid.Cid) error { - k := datastore.NewKey(i.String()) +func (st *StateStore) End(i interface{}) error { + k := toKey(i) has, err := st.ds.Has(k) if err != nil { return err @@ -70,12 +81,12 @@ func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { } // mutator func(*T) error -func (st *StateStore) Mutate(i cid.Cid, mutator interface{}) error { +func (st *StateStore) Mutate(i fmt.Stringer, mutator interface{}) error { return st.mutate(i, cborMutator(mutator)) } -func (st *StateStore) mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error { - k := datastore.NewKey(i.String()) +func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error { + k := toKey(i) has, err := st.ds.Has(k) if err != nil { return err From 080a84970ccb6589f8c1ea2d5868c6150d06a20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 13:01:16 +0100 Subject: [PATCH 108/230] api: Split api files --- api/api_common.go | 52 +++++++++++++++++++++++++ api/{api.go => api_full.go} | 75 ------------------------------------- api/api_storage.go | 46 +++++++++++++++++++++++ 3 files changed, 98 insertions(+), 75 deletions(-) create mode 100644 api/api_common.go rename api/{api.go => api_full.go} (82%) create mode 100644 api/api_storage.go diff --git a/api/api_common.go b/api/api_common.go new file mode 100644 index 000000000..376aef86d --- /dev/null +++ b/api/api_common.go @@ -0,0 +1,52 @@ +package api + +import ( + "context" + "fmt" + + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + "github.com/filecoin-project/lotus/build" +) + +type Common interface { + // Auth + AuthVerify(ctx context.Context, token string) ([]Permission, error) + AuthNew(ctx context.Context, perms []Permission) ([]byte, error) + + // network + + NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) + NetPeers(context.Context) ([]peer.AddrInfo, error) + NetConnect(context.Context, peer.AddrInfo) error + NetAddrsListen(context.Context) (peer.AddrInfo, error) + NetDisconnect(context.Context, peer.ID) error + + // ID returns peerID of libp2p node backing this API + ID(context.Context) (peer.ID, error) + + // Version provides information about API provider + Version(context.Context) (Version, error) +} + +// Version provides various build-time information +type Version struct { + Version string + + // APIVersion is a binary encoded semver version of the remote implementing + // this api + // + // See APIVersion in build/version.go + APIVersion uint32 + + // TODO: git commit / os / genesis cid? + + // Seconds + BlockDelay uint64 +} + +func (v Version) String() string { + vM, vm, vp := build.VersionInts(v.APIVersion) + return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp) +} \ No newline at end of file diff --git a/api/api.go b/api/api_full.go similarity index 82% rename from api/api.go rename to api/api_full.go index 635901ecb..e071a7066 100644 --- a/api/api.go +++ b/api/api_full.go @@ -2,46 +2,16 @@ package api import ( "context" - "fmt" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" ) -func init() { - cbor.RegisterCborType(SealedRef{}) -} - -type Common interface { - // Auth - AuthVerify(ctx context.Context, token string) ([]Permission, error) - AuthNew(ctx context.Context, perms []Permission) ([]byte, error) - - // network - - NetConnectedness(context.Context, peer.ID) (network.Connectedness, error) - NetPeers(context.Context) ([]peer.AddrInfo, error) - NetConnect(context.Context, peer.AddrInfo) error - NetAddrsListen(context.Context) (peer.AddrInfo, error) - NetDisconnect(context.Context, peer.ID) error - - // ID returns peerID of libp2p node backing this API - ID(context.Context) (peer.ID, error) - - // Version provides information about API provider - Version(context.Context) (Version, error) -} - // FullNode API is a low-level interface to the Filecoin network full node type FullNode interface { Common @@ -152,45 +122,6 @@ type FullNode interface { PaychVoucherSubmit(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) } -// StorageMiner is a low-level interface to the Filecoin network storage miner node -type StorageMiner interface { - Common - - ActorAddress(context.Context) (address.Address, error) - - // Temp api for testing - StoreGarbageData(context.Context) error - - // Get the status of a given sector by ID - SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) - - // List all staged sectors - SectorsList(context.Context) ([]uint64, error) - - SectorsRefs(context.Context) (map[string][]SealedRef, error) -} - -// Version provides various build-time information -type Version struct { - Version string - - // APIVersion is a binary encoded semver version of the remote implementing - // this api - // - // See APIVersion in build/version.go - APIVersion uint32 - - // TODO: git commit / os / genesis cid? - - // Seconds - BlockDelay uint64 -} - -func (v Version) String() string { - vM, vm, vp := build.VersionInts(v.APIVersion) - return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp) -} - type Import struct { Status filestore.Status Key cid.Cid @@ -275,12 +206,6 @@ type MinerPower struct { TotalPower types.BigInt } -type SealedRef struct { - Piece string - Offset uint64 - Size uint32 -} - type QueryOffer struct { Err string diff --git a/api/api_storage.go b/api/api_storage.go new file mode 100644 index 000000000..166f33b4a --- /dev/null +++ b/api/api_storage.go @@ -0,0 +1,46 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +type SectorState int + +const ( + Undefined SectorState = iota + + Empty // TODO: Is this useful + Packing // sector not in sealStore, and not on chain + + Unsealed // sealing / queued + PreCommitting // on chain pre-commit + PreCommitted // waiting for seed + Committing +) + +// StorageMiner is a low-level interface to the Filecoin network storage miner node +type StorageMiner interface { + Common + + ActorAddress(context.Context) (address.Address, error) + + // Temp api for testing + StoreGarbageData(context.Context) error + + // Get the status of a given sector by ID + SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) + + // List all staged sectors + SectorsList(context.Context) ([]uint64, error) + + SectorsRefs(context.Context) (map[string][]SealedRef, error) +} + +type SealedRef struct { + Piece string + Offset uint64 + Size uint32 +} From 82344649d31b63f705d672212669d0068ce45229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 14:58:48 +0100 Subject: [PATCH 109/230] Use state store for sectors --- api/api_common.go | 2 +- api/api_storage.go | 31 +++- chain/types/cbor_gen.go | 115 +++++++++++++ chain/types/tipset.go | 31 +++- gen/main.go | 11 ++ lib/statestore/store.go | 8 +- node/impl/storminer.go | 2 +- node/modules/storageminer.go | 1 + storage/cbor_gen.go | 319 +++++++++++++++++++++++++++++++++++ storage/miner.go | 39 +++-- storage/post.go | 12 +- storage/sealing.go | 236 +++++++++++--------------- storage/sector/store.go | 14 +- storage/sector_states.go | 183 ++++++++++++++++++++ 14 files changed, 830 insertions(+), 174 deletions(-) create mode 100644 storage/cbor_gen.go create mode 100644 storage/sector_states.go diff --git a/api/api_common.go b/api/api_common.go index 376aef86d..c83a4c260 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -49,4 +49,4 @@ type Version struct { func (v Version) String() string { vM, vm, vp := build.VersionInts(v.APIVersion) return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp) -} \ No newline at end of file +} diff --git a/api/api_storage.go b/api/api_storage.go index 166f33b4a..7a392455f 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -2,15 +2,17 @@ package api import ( "context" + "fmt" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" ) -type SectorState int +// alias because cbor-gen doesn't like non-alias types +type SectorState = uint64 const ( - Undefined SectorState = iota + UndefinedSectorState SectorState = iota Empty // TODO: Is this useful Packing // sector not in sealStore, and not on chain @@ -19,8 +21,33 @@ const ( PreCommitting // on chain pre-commit PreCommitted // waiting for seed Committing + Proving + + SectorNoUpdate = UndefinedSectorState ) +func SectorStateStr(s SectorState) string { + switch s { + case UndefinedSectorState: + return "UndefinedSectorState" + case Empty: + return "Empty" + case Packing: + return "Packing" + case Unsealed: + return "Unsealed" + case PreCommitting: + return "PreCommitting" + case PreCommitted: + return "PreCommitted" + case Committing: + return "Committing" + case Proving: + return "Proving" + } + return fmt.Sprintf("", s) +} + // StorageMiner is a low-level interface to the Filecoin network storage miner node type StorageMiner interface { Common diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index ab63b28a1..cb9c85959 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1470,3 +1470,118 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { t.SeqNo = extra return nil } + +func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Cids ([]cid.Cid) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Cids)))); err != nil { + return err + } + for _, v := range t.Cids { + if err := cbg.WriteCid(w, v); err != nil { + return xerrors.Errorf("failed writing cid field t.Cids: %w", err) + } + } + + // t.t.Blocks ([]*types.BlockHeader) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Blocks)))); err != nil { + return err + } + for _, v := range t.Blocks { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.Height (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Height)); err != nil { + return err + } + return nil +} + +func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Cids ([]cid.Cid) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Cids: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Cids = make([]cid.Cid, extra) + } + for i := 0; i < int(extra); i++ { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("reading cid field t.Cids failed: %w", err) + } + t.Cids[i] = c + } + + // t.t.Blocks ([]*types.BlockHeader) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Blocks: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Blocks = make([]*BlockHeader, extra) + } + for i := 0; i < int(extra); i++ { + + var v BlockHeader + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Blocks[i] = &v + } + + // t.t.Height (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Height = extra + return nil +} diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 3733a6f09..0939fef1c 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "sort" "github.com/ipfs/go-cid" @@ -20,14 +21,14 @@ type TipSet struct { // why didnt i just export the fields? Because the struct has methods with the // same names already -type expTipSet struct { +type ExpTipSet struct { Cids []cid.Cid Blocks []*BlockHeader Height uint64 } func (ts *TipSet) MarshalJSON() ([]byte, error) { - return json.Marshal(expTipSet{ + return json.Marshal(ExpTipSet{ Cids: ts.cids, Blocks: ts.blks, Height: ts.height, @@ -35,7 +36,7 @@ func (ts *TipSet) MarshalJSON() ([]byte, error) { } func (ts *TipSet) UnmarshalJSON(b []byte) error { - var ets expTipSet + var ets ExpTipSet if err := json.Unmarshal(b, &ets); err != nil { return err } @@ -50,6 +51,30 @@ func (ts *TipSet) UnmarshalJSON(b []byte) error { return nil } +func (ts *TipSet) MarshalCBOR(w io.Writer) error { + return (&ExpTipSet{ + Cids: ts.cids, + Blocks: ts.blks, + Height: ts.height, + }).MarshalCBOR(w) +} + +func (ts *TipSet) UnmarshalCBOR(r io.Reader) error { + var ets ExpTipSet + if err := ets.UnmarshalCBOR(r); err != nil { + return err + } + + ots, err := NewTipSet(ets.Blocks) + if err != nil { + return err + } + + *ts = *ots + + return nil +} + func tipsetSortFunc(blks []*BlockHeader) func(i, j int) bool { return func(i, j int) bool { ti := blks[i].LastTicket() diff --git a/gen/main.go b/gen/main.go index a588dfadb..335aa0584 100644 --- a/gen/main.go +++ b/gen/main.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storage" ) func main() { @@ -26,6 +27,7 @@ func main() { types.BlockMsg{}, types.SignedStorageAsk{}, types.StorageAsk{}, + types.ExpTipSet{}, ) if err != nil { fmt.Println(err) @@ -109,4 +111,13 @@ func main() { fmt.Println(err) os.Exit(1) } + + err = gen.WriteTupleEncodersToFile("./storage/cbor_gen.go", "storage", + storage.SealTicket{}, + storage.SectorInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 2fd4a4384..52734bd2b 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -3,11 +3,13 @@ package statestore import ( "bytes" "fmt" - "github.com/filecoin-project/lotus/lib/cborrpc" + "reflect" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "golang.org/x/xerrors" - "reflect" + + "github.com/filecoin-project/lotus/lib/cborrpc" ) type StateStore struct { @@ -81,7 +83,7 @@ func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { } // mutator func(*T) error -func (st *StateStore) Mutate(i fmt.Stringer, mutator interface{}) error { +func (st *StateStore) Mutate(i interface{}, mutator interface{}) error { return st.mutate(i, cborMutator(mutator)) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b8b101ec3..d9ee34463 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -47,7 +47,7 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { return } - if err := sm.Miner.SealSector(ctx, sectorId); err != nil { + if err := sm.Miner.SealSector(context.TODO(), sectorId); err != nil { log.Error(err) return } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 73d55c5d1..a7580f84c 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -90,6 +90,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h OnStart: func(context.Context) error { return sm.Run(ctx) }, + OnStop: sm.Stop, }) return sm, nil diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go new file mode 100644 index 000000000..518db6692 --- /dev/null +++ b/storage/cbor_gen.go @@ -0,0 +1,319 @@ +package storage + +import ( + "fmt" + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" + "io" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *SealTicket) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.BlockHeight (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.BlockHeight)); err != nil { + return err + } + + // t.t.TicketBytes ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { + return err + } + if _, err := w.Write(t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.BlockHeight (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = extra + // t.t.TicketBytes ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SectorInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{137}); err != nil { + return err + } + + // t.t.State (api.SectorState) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + return err + } + + // t.t.SectorID (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + return err + } + + // t.t.CommD ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { + return err + } + if _, err := w.Write(t.CommD); err != nil { + return err + } + + // t.t.CommR ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { + return err + } + if _, err := w.Write(t.CommR); err != nil { + return err + } + + // t.t.Ticket (storage.SealTicket) + if err := t.Ticket.MarshalCBOR(w); err != nil { + return err + } + + // t.t.PreCommitMessage (cid.Cid) + + if t.PreCommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PreCommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommitMessage: %w", err) + } + } + + // t.t.RandHeight (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.RandHeight)); err != nil { + return err + } + + // t.t.RandTs (types.TipSet) + if err := t.RandTs.MarshalCBOR(w); err != nil { + return err + } + + // t.t.CommitMessage (cid.Cid) + + if t.CommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + } + } + + return nil +} + +func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 9 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.State (api.SectorState) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = extra + // t.t.SectorID (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = extra + // t.t.CommD ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommD: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommD = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommD); err != nil { + return err + } + // t.t.CommR ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommR: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommR = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommR); err != nil { + return err + } + // t.t.Ticket (storage.SealTicket) + + { + + if err := t.Ticket.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.PreCommitMessage (cid.Cid) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) + } + + t.PreCommitMessage = &c + } + + } + // t.t.RandHeight (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.RandHeight = extra + // t.t.RandTs (types.TipSet) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.RandTs = new(types.TipSet) + if err := t.RandTs.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.CommitMessage (cid.Cid) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = &c + } + + } + return nil +} diff --git a/storage/miner.go b/storage/miner.go index bb3be8f8c..18449a42c 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,6 +2,8 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/lib/statestore" + "github.com/ipfs/go-datastore/namespace" "sync" "github.com/ipfs/go-cid" @@ -15,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sector" ) @@ -26,19 +27,23 @@ const PoStConfidence = 3 type Miner struct { api storageMinerApi events *events.Events + h host.Host + secst *sector.Store - secst *sector.Store - - maddr address.Address - + maddr address.Address worker address.Address - h host.Host - - ds datastore.Batching - - schedLk sync.Mutex + // PoSt + postLk sync.Mutex schedPost uint64 + + // Sealing + sectors *statestore.StateStore + + sectorIncoming chan *SectorInfo + sectorUpdated chan sectorUpdate + stop chan struct{} + stopped chan struct{} } type storageMinerApi interface { @@ -72,8 +77,9 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto maddr: addr, h: h, - ds: ds, secst: secst, + + sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))), }, nil } @@ -85,11 +91,18 @@ func (m *Miner) Run(ctx context.Context) error { m.events = events.NewEvents(ctx, m.api) go m.beginPosting(ctx) + go m.sectorStateLoop(ctx) return nil } -func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error { - return m.SealSector(ctx, sinfo.SectorID) +func (m *Miner) Stop(ctx context.Context) error { + close(m.stop) + select { + case <-m.stopped: + return nil + case <-ctx.Done(): + return ctx.Err() + } } func (m *Miner) runPreflightChecks(ctx context.Context) error { diff --git a/storage/post.go b/storage/post.go index ee024acbb..9d3f892a7 100644 --- a/storage/post.go +++ b/storage/post.go @@ -30,10 +30,10 @@ func (m *Miner) beginPosting(ctx context.Context) { return } - m.schedLk.Lock() + m.postLk.Lock() if m.schedPost > 0 { log.Warnf("PoSts already running %d", m.schedPost) - m.schedLk.Unlock() + m.postLk.Unlock() return } @@ -42,7 +42,7 @@ func (m *Miner) beginPosting(ctx context.Context) { ppe, _ = actors.ProvingPeriodEnd(ppe, ts.Height()+1) m.schedPost = ppe - m.schedLk.Unlock() + m.postLk.Unlock() log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime) err = m.events.ChainAt(m.computePost(m.schedPost), func(ts *types.TipSet) error { // Revert @@ -71,16 +71,16 @@ func (m *Miner) scheduleNextPost(ppe uint64) { ppe = headPPE } - m.schedLk.Lock() + m.postLk.Lock() if m.schedPost >= ppe { // this probably can't happen log.Errorw("PoSt already scheduled", "schedPost", m.schedPost, "ppe", ppe) - m.schedLk.Unlock() + m.postLk.Unlock() return } m.schedPost = ppe - m.schedLk.Unlock() + m.postLk.Unlock() log.Infow("scheduling PoSt", "post-height", ppe-build.PoStChallangeTime, "height", ts.Height(), "ppe", ppe, "proving-period", provingPeriod) diff --git a/storage/sealing.go b/storage/sealing.go index 2a3d02d7f..651dbf6ca 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,154 +2,120 @@ package storage import ( "context" - "fmt" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" - "github.com/pkg/errors" - "golang.org/x/xerrors" ) -func (m *Miner) SealSector(ctx context.Context, sid uint64) error { - log.Info("committing sector: ", sid) - - ssize, err := m.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to check out own sector size: %w", err) - } - - _ = ssize - - log.Info("performing sector replication...") - if err := m.secst.SealPreCommit(ctx, sid); err != nil { - return xerrors.Errorf("seal pre commit failed: %w", err) - } - - sinfo, err := m.secst.SectorStatus(sid) - if err != nil { - return xerrors.Errorf("failed to check status for sector %d: %w", sid, err) - } - - params := &actors.SectorPreCommitInfo{ - CommD: sinfo.CommD[:], - CommR: sinfo.CommR[:], - Epoch: sinfo.Ticket.BlockHeight, - - //DealIDs: deals, - SectorNumber: sinfo.SectorID, - } - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - return errors.Wrap(aerr, "could not serialize commit sector parameters") - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.PreCommitSector, - Params: enc, - Value: types.NewInt(0), // TODO: need to ensure sufficient collateral - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - log.Info("submitting precommit for sector: ", sid) - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return errors.Wrap(err, "pushing message to mpool") - } - - go m.waitForPreCommitMessage(context.TODO(), sinfo.SectorID, smsg.Cid()) - - // TODO: maybe return a wait channel? - return nil +type SealTicket struct { + BlockHeight uint64 + TicketBytes []byte } -func (m *Miner) waitForPreCommitMessage(ctx context.Context, sid uint64, mcid cid.Cid) { - // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts - mw, err := m.api.StateWaitMsg(ctx, mcid) - if err != nil { - return - } +type SectorInfo struct { + State api.SectorState + SectorID uint64 - if mw.Receipt.ExitCode != 0 { - log.Error("sector precommit failed: ", mw.Receipt.ExitCode) - return - } + // PreCommit + CommD []byte + CommR []byte + Ticket SealTicket - randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - log.Infof("precommit for sector %d made it on chain, will start post computation at height %d", sid, randHeight) + PreCommitMessage *cid.Cid - err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { - return m.scheduleComputeProof(ctx, sid, ts, randHeight) - }, func(ts *types.TipSet) error { - log.Warn("revert in interactive commit sector step") - return nil - }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) - if err != nil { - log.Warn("waitForPreCommitMessage ChainAt errored: ", err) - } + // PreCommitted + RandHeight uint64 + RandTs *types.TipSet + + // Committing + CommitMessage *cid.Cid } -func (m *Miner) scheduleComputeProof(ctx context.Context, sid uint64, ts *types.TipSet, rheight uint64) error { - log.Info("scheduling post computation...") +type sectorUpdate struct { + newState api.SectorState + id uint64 + err error + mut func(*SectorInfo) +} + +func (m *Miner) sectorStateLoop(ctx context.Context) { + // TODO: restore state + go func() { - rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-rheight)) - if err != nil { - log.Error(fmt.Errorf("failed to get randomness for computing seal proof: %w", err)) - return - } + defer log.Warn("quitting deal provider loop") + defer close(m.stopped) - proof, err := m.secst.SealComputeProof(ctx, sid, rheight, rand) - if err != nil { - log.Error(fmt.Errorf("computing seal proof failed: %w", err)) - return + for { + select { + case sector := <-m.sectorIncoming: + m.onSectorIncoming(sector) + case update := <-m.sectorUpdated: + m.onSectorUpdated(ctx, update) + case <-m.stop: + return + } } - - params := &actors.SectorProveCommitInfo{ - Proof: proof, - SectorID: sid, - //DealIDs: deals, - } - - _ = params - enc, aerr := actors.SerializeParams(nil) - if aerr != nil { - log.Error(errors.Wrap(aerr, "could not serialize commit sector parameters")) - return - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.ProveCommitSector, - Params: enc, - Value: types.NewInt(0), // TODO: need to ensure sufficient collateral - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - log.Error(errors.Wrap(err, "pushing message to mpool")) - } - - // TODO: now wait for this to get included and handle errors? - mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - log.Errorf("failed to wait for porep inclusion: %s", err) - return - } - - if mw.Receipt.ExitCode != 0 { - log.Error("UNHANDLED: submitting sector proof failed") - return - } - - m.beginPosting(ctx) }() - - return nil +} + +func (m *Miner) onSectorIncoming(sector *SectorInfo) { + if err := m.sectors.Begin(sector.SectorID, sector); err != nil { + // We may have re-sent the proposal + log.Errorf("deal tracking failed: %s", err) + m.failSector(sector.SectorID, err) + return + } + + go func() { + m.sectorUpdated <- sectorUpdate{ + newState: api.Unsealed, + id: sector.SectorID, + } + }() +} + +func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { + log.Infof("Sector %s updated state to %s", update.id, api.SectorStateStr(update.newState)) + var sector SectorInfo + err := m.sectors.Mutate(update.id, func(s *SectorInfo) error { + s.State = update.newState + sector = *s + return nil + }) + if update.err != nil { + log.Errorf("deal %s failed: %s", update.id, update.err) + m.failSector(update.id, update.err) + return + } + if err != nil { + m.failSector(update.id, err) + return + } + + switch update.newState { + case api.Unsealed: + m.handle(ctx, sector, m.sealPreCommit, api.PreCommitting) + case api.PreCommitting: + m.handle(ctx, sector, m.preCommit, api.PreCommitted) + case api.PreCommitted: + m.handle(ctx, sector, m.preCommitted, api.SectorNoUpdate) + case api.Committing: + m.handle(ctx, sector, m.committing, api.Proving) + } +} + +func (m *Miner) failSector(id uint64, err error) { + panic(err) // todo: better error handling strategy +} + +func (m *Miner) SealSector(ctx context.Context, sid uint64) error { + select { + case m.sectorIncoming <- &SectorInfo{ + State: api.UndefinedSectorState, + SectorID: sid, + }: + return nil + case <-ctx.Done(): + return ctx.Err() + } } diff --git a/storage/sector/store.go b/storage/sector/store.go index 855fe2c39..52c45deef 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -131,19 +131,13 @@ func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { } } -func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) error { +func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) (sectorbuilder.SealPreCommitOutput, error) { tkt, err := s.tktFn(ctx) if err != nil { - return err + return sectorbuilder.SealPreCommitOutput{}, err } - // TODO: That's not async, is it? - // - If not then we probably can drop this wait-for-seal hack below - _, err = s.sb.SealPreCommit(sectorID, *tkt) - if err != nil { - return err - } - return nil + return s.sb.SealPreCommit(sectorID, *tkt) } func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height uint64, rand []byte) ([]byte, error) { @@ -160,7 +154,7 @@ func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height ui return sco.Proof, nil } -func (s *Store) Commited() ([]sectorbuilder.SectorSealingStatus, error) { +func (s *Store) Committed() ([]sectorbuilder.SectorSealingStatus, error) { l, err := s.sb.GetAllStagedSectors() if err != nil { return nil, err diff --git a/storage/sector_states.go b/storage/sector_states.go new file mode 100644 index 000000000..ce4a8b28c --- /dev/null +++ b/storage/sector_states.go @@ -0,0 +1,183 @@ +package storage + +import ( + "context" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/pkg/errors" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" +) + +type providerHandlerFunc func(ctx context.Context, deal SectorInfo) (func(*SectorInfo), error) + +func (m *Miner) handle(ctx context.Context, sector SectorInfo, cb providerHandlerFunc, next api.SectorState) { + go func() { + mut, err := cb(ctx, sector) + + if err == nil && next == api.SectorNoUpdate { + return + } + + select { + case m.sectorUpdated <- sectorUpdate{ + newState: next, + id: sector.SectorID, + err: err, + mut: mut, + }: + case <-m.stop: + } + }() +} + +func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Infow("performing sector replication...", "sector", sector.SectorID) + sinfo, err := m.secst.SealPreCommit(ctx, sector.SectorID) + if err != nil { + return nil, xerrors.Errorf("seal pre commit failed: %w", err) + } + + return func(info *SectorInfo) { + info.CommD = sinfo.CommD[:] + info.CommR = sinfo.CommR[:] + info.Ticket = SealTicket{ + BlockHeight: sinfo.Ticket.BlockHeight, + TicketBytes: sinfo.Ticket.TicketBytes[:], + } + }, nil +} + +func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + params := &actors.SectorPreCommitInfo{ + CommD: sector.CommD, + CommR: sector.CommR, + Epoch: sector.Ticket.BlockHeight, + + SectorNumber: sector.SectorID, + } + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.PreCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + log.Info("submitting precommit for sector: ", sector.SectorID) + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return nil, xerrors.Errorf("pushing message to mpool: %w", err) + } + + return func(info *SectorInfo) { + mcid := smsg.Cid() + info.PreCommitMessage = &mcid + }, nil +} + +func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts + mw, err := m.api.StateWaitMsg(ctx, *sector.PreCommitMessage) + if err != nil { + return nil, err + } + + if mw.Receipt.ExitCode != 0 { + log.Error("sector precommit failed: ", mw.Receipt.ExitCode) + return nil, err + } + + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay + log.Infof("precommit for sector %d made it on chain, will start post computation at height %d", sector.SectorID, randHeight) + + err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { + m.sectorUpdated <- sectorUpdate{ + newState: api.Committing, + id: sector.SectorID, + mut: func(info *SectorInfo) { + info.RandHeight = randHeight + info.RandTs = ts + }, + } + + return nil + }, func(ts *types.TipSet) error { + log.Warn("revert in interactive commit sector step") + return nil + }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) + if err != nil { + log.Warn("waitForPreCommitMessage ChainAt errored: ", err) + } + + return nil, nil +} + +func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Info("scheduling seal proof computation...") + + rand, err := m.api.ChainGetRandomness(ctx, sector.RandTs, nil, int(sector.RandTs.Height()-sector.RandHeight)) + if err != nil { + return nil, xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) + } + + proof, err := m.secst.SealComputeProof(ctx, sector.SectorID, sector.RandHeight, rand) + if err != nil { + return nil, xerrors.Errorf("computing seal proof failed: %w", err) + } + + params := &actors.SectorProveCommitInfo{ + Proof: proof, + SectorID: sector.SectorID, + //DealIDs: deals, + } + + _ = params + enc, aerr := actors.SerializeParams(nil) + if aerr != nil { + return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.ProveCommitSector, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + log.Error(errors.Wrap(err, "pushing message to mpool")) + } + + // TODO: Separate state before this wait, so we persist message cid? + + mw, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return nil, xerrors.Errorf("failed to wait for porep inclusion: %w", err) + } + + if mw.Receipt.ExitCode != 0 { + log.Error("UNHANDLED: submitting sector proof failed") + return nil, xerrors.New("UNHANDLED: submitting sector proof failed") + } + + m.beginPosting(ctx) + + return func(info *SectorInfo) { + mcid := smsg.Cid() + info.CommitMessage = &mcid + }, nil +} From 7fe6563fff255a324e50c1ad1ce8949aad5a9804 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 1 Nov 2019 11:28:12 -0700 Subject: [PATCH 110/230] add state get-actor command --- cli/state.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cli/state.go b/cli/state.go index 1448703f6..91a7f9fa1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -20,6 +20,7 @@ var stateCmd = &cli.Command{ statePledgeCollateralCmd, stateListActorsCmd, stateListMinersCmd, + stateGetActorCmd, }, } @@ -251,3 +252,39 @@ var stateListActorsCmd = &cli.Command{ return nil }, } + +var stateGetActorCmd = &cli.Command{ + Name: "get-actor", + Usage: "Print actor information", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + if !cctx.Args().Present() { + return fmt.Errorf("must pass address of actor to get") + } + + addr, err := address.NewFromString(cctx.Args().First()) + if err != nil { + return err + } + + a, err := api.StateGetActor(ctx, addr, nil) + if err != nil { + return err + } + + fmt.Printf("Address:\t%s\n", addr) + fmt.Printf("Balance:\t%s\n", types.FIL(a.Balance)) + fmt.Printf("Nonce:\t\t%d\n", a.Nonce) + fmt.Printf("Code:\t\t%s\n", a.Code) + fmt.Printf("Head:\t\t%s\n", a.Head) + + return nil + }, +} From a13dfcf96ba03f231f731f2ba996c0c437772ca7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 1 Nov 2019 13:27:59 -0700 Subject: [PATCH 111/230] Add extra check sanity check for timing --- chain/actors/actor_miner.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index e18fda36c..a642beba0 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -308,6 +308,11 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC if !ok { return nil, aerrors.New(1, "no pre-commitment found for sector") } + + if us.SubmitHeight+build.InteractivePoRepDelay > vmctx.BlockHeight() { + return nil, aerrors.New(2, "too early for proof submission") + } + delete(self.PreCommittedSectors, uintToStringKey(params.SectorID)) // TODO: ensure normalization to ID address From 2fe1b5a538ad1a6369044d659c0891532dc543b4 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 1 Nov 2019 15:05:05 -0700 Subject: [PATCH 112/230] some cleanup --- storage/sealing.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/storage/sealing.go b/storage/sealing.go index 651dbf6ca..cb377172a 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,9 +2,11 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" + xerrors "golang.org/x/xerrors" ) type SealTicket struct { @@ -67,9 +69,13 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { } go func() { - m.sectorUpdated <- sectorUpdate{ + select { + case m.sectorUpdated <- sectorUpdate{ newState: api.Unsealed, id: sector.SectorID, + }: + case <-m.stop: + log.Warn("failed to send incoming sector update, miner shutting down") } }() } @@ -109,13 +115,14 @@ func (m *Miner) failSector(id uint64, err error) { } func (m *Miner) SealSector(ctx context.Context, sid uint64) error { - select { - case m.sectorIncoming <- &SectorInfo{ + si := &SectorInfo{ State: api.UndefinedSectorState, SectorID: sid, - }: + } + select { + case m.sectorIncoming <- si: return nil case <-ctx.Done(): - return ctx.Err() + return xerrors.Errorf("failed to submit sector for sealing, queue full: %w", ctx.Err()) } } From 1dcebece719b856e88d4477f2caf943e2839f60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 1 Nov 2019 23:44:55 +0100 Subject: [PATCH 113/230] Some smaller fixes --- chain/actors/actor_miner.go | 2 +- chain/types/tipset.go | 5 +++++ lib/statestore/store.go | 4 ++-- lotuspond/front/src/StorageNode.js | 13 ++++++++----- storage/cbor_gen.go | 4 ++-- storage/miner.go | 5 +++++ storage/sealing.go | 10 ++++++++-- 7 files changed, 31 insertions(+), 12 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index a642beba0..46ab1b00e 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -228,7 +228,7 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon } if params.Epoch >= vmctx.BlockHeight() { - return nil, aerrors.New(1, "sector commitment must be based off past randomness") + return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.Epoch, vmctx.BlockHeight()) } if vmctx.BlockHeight()-params.Epoch > 1000 { diff --git a/chain/types/tipset.go b/chain/types/tipset.go index 0939fef1c..1836fe813 100644 --- a/chain/types/tipset.go +++ b/chain/types/tipset.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + cbg "github.com/whyrusleeping/cbor-gen" ) var log = logging.Logger("types") @@ -52,6 +53,10 @@ func (ts *TipSet) UnmarshalJSON(b []byte) error { } func (ts *TipSet) MarshalCBOR(w io.Writer) error { + if ts == nil { + _, err := w.Write(cbg.CborNull) + return err + } return (&ExpTipSet{ Cids: ts.cids, Blocks: ts.blks, diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 52734bd2b..f578c17e3 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -74,8 +74,8 @@ func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { out := rmut.Call([]reflect.Value{state}) - if err := out[0].Interface().(error); err != nil { - return nil, err + if err := out[0].Interface(); err != nil { + return nil, err.(error) } return cborrpc.Dump(state.Interface()) diff --git a/lotuspond/front/src/StorageNode.js b/lotuspond/front/src/StorageNode.js index 741932db9..ef50c615d 100644 --- a/lotuspond/front/src/StorageNode.js +++ b/lotuspond/front/src/StorageNode.js @@ -9,12 +9,15 @@ const stateGettingToken = 'getting-token' let sealCodes = [ "Unknown", - "Pending", + "AcceptingPieces", + "Committed", + "Committing", + "CommittingPaused", "Failed", - "Sealing", - "Sealed", - "Paused", - "ReadyForSealing", + "FullyPacked", + "PreCommitted", + "PreCommitting", + "PreCommittingPaused", ] class StorageNode extends React.Component { diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 518db6692..38e70c020 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -90,7 +90,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.State (api.SectorState) + // t.t.State (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { return err } @@ -173,7 +173,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.State (api.SectorState) + // t.t.State (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { diff --git a/storage/miner.go b/storage/miner.go index 18449a42c..4cec6c94b 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -80,6 +80,11 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto secst: secst, sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))), + + sectorIncoming: make(chan *SectorInfo), + sectorUpdated: make(chan sectorUpdate), + stop: make(chan struct{}), + stopped: make(chan struct{}), }, nil } diff --git a/storage/sealing.go b/storage/sealing.go index cb377172a..d1883be3d 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -81,15 +81,18 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { } func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { - log.Infof("Sector %s updated state to %s", update.id, api.SectorStateStr(update.newState)) + log.Infof("Sector %d updated state to %s", update.id, api.SectorStateStr(update.newState)) var sector SectorInfo err := m.sectors.Mutate(update.id, func(s *SectorInfo) error { s.State = update.newState + if update.mut != nil { + update.mut(s) + } sector = *s return nil }) if update.err != nil { - log.Errorf("deal %s failed: %s", update.id, update.err) + log.Errorf("sector %d failed: %s", update.id, update.err) m.failSector(update.id, update.err) return } @@ -111,10 +114,13 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { } func (m *Miner) failSector(id uint64, err error) { + log.Error(err) panic(err) // todo: better error handling strategy } func (m *Miner) SealSector(ctx context.Context, sid uint64) error { + log.Infof("Begin sealing sector %d", sid) + si := &SectorInfo{ State: api.UndefinedSectorState, SectorID: sid, From 3bc4a5dddf989b8e4fc2d88d06301d111eabea77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Nov 2019 00:43:54 +0100 Subject: [PATCH 114/230] More progress on fixing storing garbage data --- build/params.go | 2 +- chain/actors/actor_miner.go | 8 +- chain/actors/cbor_gen.go | 109 +++++++++++++++++++++++++++ gen/main.go | 1 + node/impl/storminer.go | 29 +------- node/modules/storageminer.go | 2 +- storage/garbage.go | 140 +++++++++++++++++++++++++++++++++++ storage/miner.go | 1 + storage/sector_states.go | 10 ++- 9 files changed, 266 insertions(+), 36 deletions(-) create mode 100644 storage/garbage.go diff --git a/build/params.go b/build/params.go index f69fb05ef..ef55c01bd 100644 --- a/build/params.go +++ b/build/params.go @@ -37,7 +37,7 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // Consensus / Network // Seconds -const BlockDelay = 10 +const BlockDelay = 2 // Seconds const AllowableClockDrift = BlockDelay * 2 diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 46ab1b00e..010d32729 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -227,11 +227,11 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon return nil, err } - if params.Epoch >= vmctx.BlockHeight() { - return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.Epoch, vmctx.BlockHeight()) + if params.Epoch >= vmctx.BlockHeight()+build.SealRandomnessLookback { + return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.Epoch, vmctx.BlockHeight()+build.SealRandomnessLookback) } - if vmctx.BlockHeight()-params.Epoch > 1000 { + if vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback > 1000 { return nil, aerrors.New(2, "sector commitment must be recent enough") } @@ -318,7 +318,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC // TODO: ensure normalization to ID address maddr := vmctx.Message().To - ticket, err := vmctx.GetRandomness(us.TicketEpoch) + ticket, err := vmctx.GetRandomness(us.TicketEpoch - build.SealRandomnessLookback) if err != nil { return nil, aerrors.Wrap(err, "failed to get ticket randomness") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 63202da1a..03a59bd4a 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3771,3 +3771,112 @@ func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error { t.SectorSize = extra return nil } + +func (t *SectorProveCommitInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Proof ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + + // t.t.SectorID (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Proof ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.t.SectorID (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = extra + // t.t.DealIDs ([]uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + return nil +} diff --git a/gen/main.go b/gen/main.go index 335aa0584..2d8d10e26 100644 --- a/gen/main.go +++ b/gen/main.go @@ -91,6 +91,7 @@ func main() { actors.ProcessStorageDealsPaymentParams{}, actors.OnChainDeal{}, actors.ComputeDataCommitmentParams{}, + actors.SectorProveCommitInfo{}, ) if err != nil { fmt.Println(err) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index d9ee34463..e6bae6a22 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -2,9 +2,6 @@ package impl import ( "context" - "fmt" - "io" - "math/rand" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" @@ -12,8 +9,6 @@ import ( "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" - - "golang.org/x/xerrors" ) type StorageMinerAPI struct { @@ -25,6 +20,7 @@ type StorageMinerAPI struct { SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner + Full api.FullNode } func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { @@ -32,28 +28,7 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { - ssize, err := sm.Miner.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) - } - go func() { - size := sectorbuilder.UserBytesForSectorSize(ssize) - - // TODO: create a deal - name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) - if err != nil { - log.Error(err) - return - } - - if err := sm.Miner.SealSector(context.TODO(), sectorId); err != nil { - log.Error(err) - return - } - }() - - return err + return sm.Miner.StoreGarbageData(ctx) } func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index a7580f84c..b95500305 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -184,7 +184,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn { } return §orbuilder.SealTicket{ - BlockHeight: ts.Height() - build.SealRandomnessLookback, + BlockHeight: ts.Height(), TicketBytes: tkt, }, nil } diff --git a/storage/garbage.go b/storage/garbage.go new file mode 100644 index 000000000..d1f396299 --- /dev/null +++ b/storage/garbage.go @@ -0,0 +1,140 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "math" + "math/rand" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +func (m *Miner) StoreGarbageData(_ context.Context) error { + ctx := context.TODO() + ssize, err := m.SectorSize(ctx) + if err != nil { + return xerrors.Errorf("failed to get miner sector size: %w", err) + } + go func() { + size := sectorbuilder.UserBytesForSectorSize(ssize) + + /*// Add market funds + smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: m.worker, + Value: types.NewInt(size), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + log.Error(err) + return + } + + r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + log.Error(err) + return + } + + if r.Receipt.ExitCode != 0 { + log.Error(xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)) + return + }*/ + // Publish a deal + + // TODO: Maybe cache + commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) + if err != nil { + log.Error(err) + return + } + + sdp := actors.StorageDealProposal{ + PieceRef: commP[:], + PieceSize: size, + PieceSerialization: actors.SerializationUnixFSv0, + Client: m.worker, + Provider: m.worker, + ProposalExpiration: math.MaxUint64, + Duration: math.MaxUint64 / 2, // /2 because overflows + StoragePricePerEpoch: types.NewInt(0), + StorageCollateral: types.NewInt(0), + ProposerSignature: nil, + } + + if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil { + log.Error(xerrors.Errorf("signing storage deal failed: ", err)) + return + } + + storageDeal := actors.StorageDeal{ + Proposal: sdp, + } + if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil { + log.Error(xerrors.Errorf("signing storage deal failed: ", err)) + return + } + + params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: []actors.StorageDeal{storageDeal}, + }) + if err != nil { + log.Error(xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)) + } + + // TODO: We may want this to happen after fetching data + smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: m.worker, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + Params: params, + }) + if err != nil { + log.Error(err) + return + } + r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + log.Error(err) + return + } + if r.Receipt.ExitCode != 0 { + log.Error(xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)) + } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + log.Error(err) + return + } + if len(resp.DealIDs) != 1 { + log.Error("got unexpected number of DealIDs from") + return + } + + name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) + sectorId, err := m.secst.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), resp.DealIDs[0]) + if err != nil { + log.Error(err) + return + } + + if err := m.SealSector(context.TODO(), sectorId); err != nil { + log.Error(err) + return + } + }() + + return err +} diff --git a/storage/miner.go b/storage/miner.go index 4cec6c94b..ae93e78f5 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -67,6 +67,7 @@ type storageMinerApi interface { ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error) ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error) + WalletSign(context.Context, address.Address, []byte) (*types.Signature, error) WalletBalance(context.Context, address.Address) (types.BigInt, error) WalletHas(context.Context, address.Address) (bool, error) } diff --git a/storage/sector_states.go b/storage/sector_states.go index ce4a8b28c..3e8a0f67a 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -135,14 +135,18 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector return nil, xerrors.Errorf("computing seal proof failed: %w", err) } + deals, err := m.secst.DealsForCommit(sector.SectorID) + if err != nil { + return nil, err + } + params := &actors.SectorProveCommitInfo{ Proof: proof, SectorID: sector.SectorID, - //DealIDs: deals, + DealIDs: deals, } - _ = params - enc, aerr := actors.SerializeParams(nil) + enc, aerr := actors.SerializeParams(params) if aerr != nil { return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr) } From 68de42df8cc6336a67c85522e34365513000ab24 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 1 Nov 2019 23:30:06 -0700 Subject: [PATCH 115/230] increase actor error log verbosity --- chain/vm/vm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 38831043b..d090e5f8d 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -462,7 +462,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet, return nil, xerrors.Errorf("fatal error: %w", actorErr) } if actorErr != nil { - log.Warnf("[from=%s,n=%d,h=%d] Send actor error: %+v", msg.From, msg.Nonce, vm.blockHeight, actorErr) + log.Warnf("[from=%s,to=%s,n=%d,m=%d,h=%d] Send actor error: %+v", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr) } var errcode uint8 From be8fe54ea3b4a24278a2111e9d7082e7b739d710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Nov 2019 15:13:21 +0100 Subject: [PATCH 116/230] Set correct Provider addr in StoreGarbage --- chain/vm/invoker.go | 4 +++- node/impl/storminer.go | 2 +- storage/garbage.go | 2 +- storage/sector_states.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/vm/invoker.go b/chain/vm/invoker.go index decc590b7..3fa02d4c1 100644 --- a/chain/vm/invoker.go +++ b/chain/vm/invoker.go @@ -2,6 +2,7 @@ package vm import ( "bytes" + "encoding/hex" "fmt" "reflect" @@ -43,7 +44,8 @@ func (inv *invoker) Invoke(act *types.Actor, vmctx types.VMContext, method uint6 code, ok := inv.builtInCode[act.Code] if !ok { - return nil, aerrors.Newf(255, "no code for actor %s", act.Code) + log.Errorf("no code for actor %s", act.Code) + return nil, aerrors.Newf(255, "no code for actor %s(%d)(%s)", act.Code, method, hex.EncodeToString(params)) } if method >= uint64(len(code)) || code[method] == nil { return nil, aerrors.Newf(255, "no method %d on actor", method) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e6bae6a22..dbbee1ca1 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -20,7 +20,7 @@ type StorageMinerAPI struct { SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner - Full api.FullNode + Full api.FullNode } func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { diff --git a/storage/garbage.go b/storage/garbage.go index d1f396299..24af346ec 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -63,7 +63,7 @@ func (m *Miner) StoreGarbageData(_ context.Context) error { PieceSize: size, PieceSerialization: actors.SerializationUnixFSv0, Client: m.worker, - Provider: m.worker, + Provider: m.maddr, ProposalExpiration: math.MaxUint64, Duration: math.MaxUint64 / 2, // /2 because overflows StoragePricePerEpoch: types.NewInt(0), diff --git a/storage/sector_states.go b/storage/sector_states.go index 3e8a0f67a..a3e28ddb0 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -143,7 +143,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector params := &actors.SectorProveCommitInfo{ Proof: proof, SectorID: sector.SectorID, - DealIDs: deals, + DealIDs: deals, } enc, aerr := actors.SerializeParams(params) From c550e030b189b7752f6cb7fa57ce719b5d2bbbcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Nov 2019 16:07:18 +0100 Subject: [PATCH 117/230] INTERACTIVE PROEP IS ALIVE --- chain/actors/actor_miner.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 1 + lib/sectorbuilder/sectorbuilder_test.go | 2 +- storage/sector/store.go | 1 + storage/sector_states.go | 4 ++-- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 010d32729..2ec36a253 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -344,7 +344,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { - return nil, aerrors.New(2, "bad proof!") + return nil, aerrors.Newf(2, "bad proof! (t:%x; s:%x(%d); p:%x)", ticket, seed, us.SubmitHeight+build.InteractivePoRepDelay, params.Proof) } // Note: There must exist a unique index in the miner's sector set for each diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index bd57a7bda..7d903b2c1 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -140,6 +140,7 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address copy(commDa[:], commD) copy(ticketa[:], ticket) copy(seeda[:], seed) + seeda[31] = 0 // Temp Fr hack proverIDa := addressToProverID(proverID) return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof) diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 3c86bb6ef..acddfe305 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -69,7 +69,7 @@ func TestSealAndVerify(t *testing.T) { seed := sectorbuilder.SealSeed{ BlockHeight: 15, - TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8}, + TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 0}, } sco, err := sb.SealCommit(sid, seed) diff --git a/storage/sector/store.go b/storage/sector/store.go index 52c45deef..d470a5239 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -143,6 +143,7 @@ func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) (sectorbuild func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height uint64, rand []byte) ([]byte, error) { var tick [32]byte copy(tick[:], rand) + tick[31] = 0 sco, err := s.sb.SealCommit(sectorID, sectorbuilder.SealSeed{ BlockHeight: height, diff --git a/storage/sector_states.go b/storage/sector_states.go index a3e28ddb0..cbdc68a26 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -97,7 +97,7 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect return nil, err } - randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay + randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - 1 // -1 because of how the messages are applied log.Infof("precommit for sector %d made it on chain, will start post computation at height %d", sector.SectorID, randHeight) err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { @@ -174,7 +174,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector } if mw.Receipt.ExitCode != 0 { - log.Error("UNHANDLED: submitting sector proof failed") + log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, rand, sector.RandHeight, params.Proof) return nil, xerrors.New("UNHANDLED: submitting sector proof failed") } From 2a695611d5b4257552ddb92b3b9aa0998195825e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 3 Nov 2019 09:21:58 +0100 Subject: [PATCH 118/230] Update sectorbuilder --- extern/go-bls-sigs | 2 +- extern/go-sectorbuilder | 2 +- lib/sectorbuilder/sectorbuilder.go | 1 - lib/sectorbuilder/sectorbuilder_test.go | 4 ++-- storage/sector/store.go | 1 - 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/extern/go-bls-sigs b/extern/go-bls-sigs index 10fef3cbf..98479d3c7 160000 --- a/extern/go-bls-sigs +++ b/extern/go-bls-sigs @@ -1 +1 @@ -Subproject commit 10fef3cbfa8670d9e9cdffb0cd8dfd4f6f6fef07 +Subproject commit 98479d3c79620f18783da0c2c6a15f8b8eb4fa2e diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 06775780d..e95a86c6d 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 06775780d8e4db7ba123a7c76073e10d7785b076 +Subproject commit e95a86c6dac7376dc78550f253ed002455553997 diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 7d903b2c1..bd57a7bda 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -140,7 +140,6 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address copy(commDa[:], commD) copy(ticketa[:], ticket) copy(seeda[:], seed) - seeda[31] = 0 // Temp Fr hack proverIDa := addressToProverID(proverID) return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof) diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index acddfe305..d17285776 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -15,7 +15,7 @@ import ( const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - t.Skip("this is slow") + //t.Skip("this is slow") build.SectorSizes = []uint64{sectorSize} if err := build.GetParams(true); err != nil { @@ -69,7 +69,7 @@ func TestSealAndVerify(t *testing.T) { seed := sectorbuilder.SealSeed{ BlockHeight: 15, - TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 0}, + TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, }, } sco, err := sb.SealCommit(sid, seed) diff --git a/storage/sector/store.go b/storage/sector/store.go index d470a5239..52c45deef 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -143,7 +143,6 @@ func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) (sectorbuild func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height uint64, rand []byte) ([]byte, error) { var tick [32]byte copy(tick[:], rand) - tick[31] = 0 sco, err := s.sb.SealCommit(sectorID, sectorbuilder.SealSeed{ BlockHeight: height, From 3ff9b2fab4fc257171b918ad5176f3445f8ceb59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Nov 2019 13:20:01 +0100 Subject: [PATCH 119/230] pond: Pass os env --- README.md | 3 +++ lotuspond/spawn.go | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c3c710864..e40dd3038 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ In order to run lotus, please do the following: - bzr (some go dependency needs this) - jq - pkg-config +- opencl-headers +- opencl-driver +- opencl-icd-loader 2. Clone this repo & `cd` into it diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index c79d736f7..b58e0e7db 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) { cmd := exec.Command("./lotus", "daemon", "--bootstrap=false", genParam, "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = []string{"LOTUS_PATH=" + dir} + cmd.Env = append(os.Environ(), "LOTUS_PATH=" + dir) if err := cmd.Start(); err != nil { return nodeInfo{}, err } @@ -112,7 +112,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd := exec.Command("./lotus-storage-miner", initArgs...) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile) cmd.Stdout = io.MultiWriter(os.Stdout, logfile) - cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo} + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo) if err := cmd.Run(); err != nil { return nodeInfo{}, err } @@ -124,7 +124,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd = exec.Command("./lotus-storage-miner", "run", "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo} + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo) if err := cmd.Start(); err != nil { return nodeInfo{}, err } From 13da5a596625319db2addefbeebab362b29d2053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Nov 2019 17:47:08 +0100 Subject: [PATCH 120/230] Put WorkerThreads on sectorbuilder.Config --- cmd/lotus-storage-miner/run.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 14 ++++++++------ lib/sectorbuilder/sectorbuilder_test.go | 11 +++++++---- lotuspond/spawn.go | 6 +++--- node/impl/storminer.go | 2 +- node/modules/storageminer.go | 17 ++++++++++++----- node/node_test.go | 2 +- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index d0806eb16..eff0cbe10 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -77,7 +77,7 @@ var runCmd = &cli.Command{ } return lr.SetAPIEndpoint(apima) }), - node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(storageRepoPath)), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(storageRepoPath)), node.Override(new(api.FullNode), nodeApi), ) if err != nil { diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index bd57a7bda..40e88ccc6 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -40,20 +40,22 @@ type SectorBuilder struct { handle unsafe.Pointer } -type SectorBuilderConfig struct { - SectorSize uint64 - Miner address.Address +type Config struct { + SectorSize uint64 + Miner address.Address + + WorkerThreads uint8 + CacheDir string SealedDir string StagedDir string MetadataDir string } -func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { +func New(cfg *Config) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) - nemWorkerThreads := uint8(5) // TODO: from config - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, nemWorkerThreads) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) if err != nil { return nil, err } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index d17285776..ad2d8ebc2 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -37,13 +37,16 @@ func TestSealAndVerify(t *testing.T) { sealed := filepath.Join(dir, "sealed") staging := filepath.Join(dir, "staging") - sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ - SectorSize: sectorSize, + sb, err := sectorbuilder.New(§orbuilder.Config{ + SectorSize: sectorSize, + Miner: addr, + + WorkerThreads: 1, + CacheDir: cache, SealedDir: sealed, StagedDir: staging, MetadataDir: metadata, - Miner: addr, }) if err != nil { t.Fatal(err) @@ -69,7 +72,7 @@ func TestSealAndVerify(t *testing.T) { seed := sectorbuilder.SealSeed{ BlockHeight: 15, - TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, }, + TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}, } sco, err := sb.SealCommit(sid, seed) diff --git a/lotuspond/spawn.go b/lotuspond/spawn.go index b58e0e7db..eac9ea418 100644 --- a/lotuspond/spawn.go +++ b/lotuspond/spawn.go @@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) { cmd := exec.Command("./lotus", "daemon", "--bootstrap=false", genParam, "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = append(os.Environ(), "LOTUS_PATH=" + dir) + cmd.Env = append(os.Environ(), "LOTUS_PATH="+dir) if err := cmd.Start(); err != nil { return nodeInfo{}, err } @@ -112,7 +112,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd := exec.Command("./lotus-storage-miner", initArgs...) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile) cmd.Stdout = io.MultiWriter(os.Stdout, logfile) - cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo) + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) if err := cmd.Run(); err != nil { return nodeInfo{}, err } @@ -124,7 +124,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) { cmd = exec.Command("./lotus-storage-miner", "run", "--api", fmt.Sprintf("%d", 2500+id)) cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw) cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw) - cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo) + cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo) if err := cmd.Start(); err != nil { return nodeInfo{}, err } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index dbbee1ca1..b9efe9f2f 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -14,7 +14,7 @@ import ( type StorageMinerAPI struct { CommonAPI - SectorBuilderConfig *sectorbuilder.SectorBuilderConfig + SectorBuilderConfig *sectorbuilder.Config SectorBuilder *sectorbuilder.SectorBuilder Sectors *sector.Store SectorBlocks *sectorblocks.SectorBlocks diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index b95500305..26f70cdb6 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -2,6 +2,7 @@ package modules import ( "context" + "math" "path/filepath" "github.com/ipfs/go-bitswap" @@ -38,8 +39,8 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { return address.NewFromBytes(maddrb) } -func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { - return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) { +func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) { + return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) { minerAddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -55,14 +56,20 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod return nil, err } + if threads > math.MaxUint8 { + return nil, xerrors.Errorf("too many sectorbuilder threads specified: %d, max allowed: %d", threads, math.MaxUint8) + } + cache := filepath.Join(sp, "cache") metadata := filepath.Join(sp, "meta") sealed := filepath.Join(sp, "sealed") staging := filepath.Join(sp, "staging") - sb := §orbuilder.SectorBuilderConfig{ - Miner: minerAddr, - SectorSize: ssize, + sb := §orbuilder.Config{ + Miner: minerAddr, + SectorSize: ssize, + WorkerThreads: uint8(threads), + CacheDir: cache, MetadataDir: metadata, SealedDir: sealed, diff --git a/node/node_test.go b/node/node_test.go index 06f1fc1c6..e361bb5fa 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -93,7 +93,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Repo(r), node.Test(), - node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(secbpath)), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath)), node.Override(new(api.FullNode), tnd), ) require.NoError(t, err) From c76ce2a2eb4842a07f02f233ed43b8ad9292d5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 4 Nov 2019 18:36:29 +0100 Subject: [PATCH 121/230] rate-limit some sectorbuilder ops --- cmd/lotus-storage-miner/run.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 37 +++++++++++++++++++++++++++++- node/node_test.go | 2 +- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index eff0cbe10..1b2a53a80 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -77,7 +77,7 @@ var runCmd = &cli.Command{ } return lr.SetAPIEndpoint(apima) }), - node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(storageRepoPath)), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(storageRepoPath, 5)), // TODO: grab worker count from config node.Override(new(api.FullNode), nodeApi), ) if err != nil { diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 40e88ccc6..2e6c67ba6 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -8,12 +8,14 @@ import ( "unsafe" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - logging "github.com/ipfs/go-log" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" ) +const PoStReservedWorkers = 1 + var log = logging.Logger("sectorbuilder") type SectorSealingStatus = sectorbuilder.SectorSealingStatus @@ -38,6 +40,8 @@ const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer + + rateLimit chan struct{} } type Config struct { @@ -53,6 +57,10 @@ type Config struct { } func New(cfg *Config) (*SectorBuilder, error) { + if cfg.WorkerThreads <= PoStReservedWorkers { + return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers + 1, cfg.WorkerThreads) + } + proverId := addressToProverID(cfg.Miner) sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) @@ -62,9 +70,21 @@ func New(cfg *Config) (*SectorBuilder, error) { return &SectorBuilder{ handle: sbp, + rateLimit: make(chan struct{}, cfg.WorkerThreads - PoStReservedWorkers), }, nil } +func (sb *SectorBuilder) rlimit() func() { + if cap(sb.rateLimit) == len(sb.rateLimit) { + log.Warn("rate-limiting sectorbuilder call") + } + sb.rateLimit <- struct{}{} + + return func() { + <-sb.rateLimit + } +} + func addressToProverID(a address.Address) [32]byte { var proverId [32]byte copy(proverId[:], a.Payload()) @@ -81,6 +101,9 @@ func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Rea return 0, err } + ret := sb.rlimit() + defer ret() + sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f) if err != nil { return 0, err @@ -91,18 +114,30 @@ func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Rea // TODO: should *really really* return an io.ReadCloser func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, error) { + ret := sb.rlimit() + defer ret() + return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket) (SealPreCommitOutput, error) { + ret := sb.rlimit() + defer ret() + return sectorbuilder.SealPreCommit(sb.handle, sectorID, ticket) } func (sb *SectorBuilder) SealCommit(sectorID uint64, seed SealSeed) (SealCommitOutput, error) { + ret := sb.rlimit() + defer ret() + return sectorbuilder.SealCommit(sb.handle, sectorID, seed) } func (sb *SectorBuilder) ResumeSealCommit(sectorID uint64) (SealCommitOutput, error) { + ret := sb.rlimit() + defer ret() + return sectorbuilder.ResumeSealCommit(sb.handle, sectorID) } diff --git a/node/node_test.go b/node/node_test.go index e361bb5fa..60ffc59fc 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -93,7 +93,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Repo(r), node.Test(), - node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath)), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)), node.Override(new(api.FullNode), tnd), ) require.NoError(t, err) From 83868d3eedf7bf57fcd347c1bed98ccbcbe6bc6f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 4 Nov 2019 11:03:11 -0800 Subject: [PATCH 122/230] fix serialization of paych vouchers --- chain/types/cbor_gen.go | 2 +- cli/client.go | 9 +- gen/main.go | 10 ++ go.mod | 2 +- go.sum | 4 +- node/impl/paych/paych.go | 4 +- paych/cbor_gen.go | 231 +++++++++++++++++++++++++++++++++++++++ paych/store.go | 19 ++-- 8 files changed, 259 insertions(+), 22 deletions(-) create mode 100644 paych/cbor_gen.go diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 9cf6ec913..ab63b28a1 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - "github.com/ipfs/go-cid" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/cli/client.go b/cli/client.go index 7f9771a3d..2ac4b3db3 100644 --- a/cli/client.go +++ b/cli/client.go @@ -229,11 +229,12 @@ var clientRetrieveCmd = &cli.Command{ order := offers[0].Order() order.Client = payer - err = api.ClientRetrieve(ctx, order, cctx.Args().Get(1)) - if err == nil { - fmt.Println("Success") + if err := api.ClientRetrieve(ctx, order, cctx.Args().Get(1)); err != nil { + return err } - return err + + fmt.Println("Success") + return nil }, } diff --git a/gen/main.go b/gen/main.go index 127336a0b..f7b4c68ef 100644 --- a/gen/main.go +++ b/gen/main.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/paych" ) func main() { @@ -32,6 +33,15 @@ func main() { os.Exit(1) } + err = gen.WriteTupleEncodersToFile("./paych/cbor_gen.go", "paych", + paych.VoucherInfo{}, + paych.ChannelInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + /* err = gen.WriteTupleEncodersToFile("./chain/cbor_gen.go", "chain", chain.BlockSyncRequest{}, diff --git a/go.mod b/go.mod index 67f47edc4..4ad2e0779 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86 + github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.0 diff --git a/go.sum b/go.sum index fd9e20aa9..518e5e1eb 100644 --- a/go.sum +++ b/go.sum @@ -628,8 +628,8 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86 h1:cE8mRdI9JQAheSlIAkjJIpdAOPjYOzxSADaro6LNHnY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191001154818-b4b5288fcb86/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c h1:P3leakVQLSvS0b953QG0BcuoCul79KTNpHbE04Bxhmc= +github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index b89980a00..e4ae9e23f 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -162,12 +162,12 @@ func (a *PaychAPI) PaychVoucherCreate(ctx context.Context, pch address.Address, func (a *PaychAPI) paychVoucherCreate(ctx context.Context, pch address.Address, voucher types.SignedVoucher) (*types.SignedVoucher, error) { ci, err := a.PaychMgr.GetChannelInfo(pch) if err != nil { - return nil, err + return nil, xerrors.Errorf("get channel info: %w", err) } nonce, err := a.PaychMgr.NextNonceForLane(ctx, pch, voucher.Lane) if err != nil { - return nil, err + return nil, xerrors.Errorf("getting next nonce for lane: %w", err) } sv := &voucher diff --git a/paych/cbor_gen.go b/paych/cbor_gen.go new file mode 100644 index 000000000..5eac7c08e --- /dev/null +++ b/paych/cbor_gen.go @@ -0,0 +1,231 @@ +package paych + +import ( + "fmt" + "io" + + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Voucher (types.SignedVoucher) + if err := t.Voucher.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Proof ([]uint8) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + return nil +} + +func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Voucher (types.SignedVoucher) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Voucher = new(types.SignedVoucher) + if err := t.Voucher.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.Proof ([]uint8) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + return nil +} + +func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{134}); err != nil { + return err + } + + // t.t.Channel (address.Address) + if err := t.Channel.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Control (address.Address) + if err := t.Control.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Target (address.Address) + if err := t.Target.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Direction (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Direction)); err != nil { + return err + } + + // t.t.Vouchers ([]*paych.VoucherInfo) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { + return err + } + for _, v := range t.Vouchers { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.NextLane (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextLane)); err != nil { + return err + } + return nil +} + +func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Channel (address.Address) + + { + + if err := t.Channel.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Control (address.Address) + + { + + if err := t.Control.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Target (address.Address) + + { + + if err := t.Target.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Direction (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Direction = extra + // t.t.Vouchers ([]*paych.VoucherInfo) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Vouchers = make([]*VoucherInfo, extra) + } + for i := 0; i < int(extra); i++ { + + var v VoucherInfo + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = &v + } + + // t.t.NextLane (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NextLane = extra + return nil +} diff --git a/paych/store.go b/paych/store.go index cc78b4b5b..2b2c1e00d 100644 --- a/paych/store.go +++ b/paych/store.go @@ -1,6 +1,7 @@ package paych import ( + "bytes" "errors" "fmt" "strings" @@ -9,7 +10,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" dsq "github.com/ipfs/go-datastore/query" - cbor "github.com/ipfs/go-ipld-cbor" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" @@ -19,11 +19,6 @@ import ( var ErrChannelNotTracked = errors.New("channel not tracked") -func init() { - cbor.RegisterCborType(VoucherInfo{}) - cbor.RegisterCborType(ChannelInfo{}) -} - type Store struct { lk sync.Mutex // TODO: this can be split per paych @@ -52,7 +47,7 @@ type ChannelInfo struct { Control address.Address Target address.Address - Direction int + Direction uint64 Vouchers []*VoucherInfo NextLane uint64 } @@ -64,12 +59,12 @@ func dskeyForChannel(addr address.Address) datastore.Key { func (ps *Store) putChannelInfo(ci *ChannelInfo) error { k := dskeyForChannel(ci.Channel) - b, err := cbor.DumpObject(ci) - if err != nil { + buf := new(bytes.Buffer) + if err := ci.MarshalCBOR(buf); err != nil { return err } - return ps.ds.Put(k, b) + return ps.ds.Put(k, buf.Bytes()) } func (ps *Store) getChannelInfo(addr address.Address) (*ChannelInfo, error) { @@ -84,7 +79,7 @@ func (ps *Store) getChannelInfo(addr address.Address) (*ChannelInfo, error) { } var ci ChannelInfo - if err := cbor.DecodeInto(b, &ci); err != nil { + if err := ci.UnmarshalCBOR(bytes.NewReader(b)); err != nil { return nil, err } @@ -161,7 +156,7 @@ func (ps *Store) findChan(filter func(*ChannelInfo) bool) (address.Address, erro return address.Undef, err } - if err := cbor.DecodeInto(res.Value, &ci); err != nil { + if err := ci.UnmarshalCBOR(bytes.NewReader(res.Value)); err != nil { return address.Undef, err } From 9a398c6260bc41629ff70b90234578b3f748d784 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 4 Nov 2019 19:42:13 -0800 Subject: [PATCH 123/230] Track down all the uses of cboripld and eliminate them --- api/cbor_gen.go | 129 +++++ chain/actors/cbor_gen.go | 624 ++++++++++++------------ chain/deals/cbor_gen.go | 144 +++--- chain/deals/state_store.go | 1 + chain/types/cbor_gen.go | 292 +++++------ chain/types/voucher.go | 5 +- gen/main.go | 25 + go.mod | 4 +- go.sum | 58 ++- lib/sectorbuilder/mock.go | 42 ++ lib/sectorbuilder/sectorbuilder_test.go | 26 +- node/hello/hello.go | 1 + node/impl/client/client.go | 3 +- paych/cbor_gen.go | 40 +- retrieval/cbor_gen.go | 473 ++++++++++++++++++ retrieval/client.go | 15 +- retrieval/miner.go | 14 +- retrieval/types.go | 14 +- 18 files changed, 1300 insertions(+), 610 deletions(-) create mode 100644 api/cbor_gen.go create mode 100644 lib/sectorbuilder/mock.go create mode 100644 retrieval/cbor_gen.go diff --git a/api/cbor_gen.go b/api/cbor_gen.go new file mode 100644 index 000000000..6f71bbaed --- /dev/null +++ b/api/cbor_gen.go @@ -0,0 +1,129 @@ +package api + +import ( + "fmt" + "io" + + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Channel (address.Address) (struct) + if err := t.Channel.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ChannelMessage (cid.Cid) (struct) + + if t.ChannelMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.ChannelMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.ChannelMessage: %w", err) + } + } + + // t.t.Vouchers ([]*types.SignedVoucher) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { + return err + } + for _, v := range t.Vouchers { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Channel (address.Address) (struct) + + { + + if err := t.Channel.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ChannelMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err) + } + + t.ChannelMessage = &c + } + + } + // t.t.Vouchers ([]*types.SignedVoucher) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Vouchers = make([]*types.SignedVoucher, extra) + } + for i := 0; i < int(extra); i++ { + + var v types.SignedVoucher + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = &v + } + + return nil +} diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 5444dafb0..304722e45 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -24,14 +24,14 @@ func (t *InitActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.AddressMap (cid.Cid) + // t.t.AddressMap (cid.Cid) (struct) if err := cbg.WriteCid(w, t.AddressMap); err != nil { return xerrors.Errorf("failed to write cid field t.AddressMap: %w", err) } - // t.t.NextID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextID)); err != nil { + // t.t.NextID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextID))); err != nil { return err } return nil @@ -52,7 +52,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.AddressMap (cid.Cid) + // t.t.AddressMap (cid.Cid) (struct) { @@ -64,7 +64,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { t.AddressMap = c } - // t.t.NextID (uint64) + // t.t.NextID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -73,7 +73,7 @@ func (t *InitActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextID = extra + t.NextID = uint64(extra) return nil } @@ -86,13 +86,13 @@ func (t *ExecParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Code); err != nil { return xerrors.Errorf("failed to write cid field t.Code: %w", err) } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -117,7 +117,7 @@ func (t *ExecParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) { @@ -129,7 +129,7 @@ func (t *ExecParams) UnmarshalCBOR(r io.Reader) error { t.Code = c } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -158,7 +158,7 @@ func (t *AccountActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Address (address.Address) + // t.t.Address (address.Address) (struct) if err := t.Address.MarshalCBOR(w); err != nil { return err } @@ -180,7 +180,7 @@ func (t *AccountActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Address (address.Address) + // t.t.Address (address.Address) (struct) { @@ -201,66 +201,66 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Info (cid.Cid) + // t.t.Info (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Info); err != nil { return xerrors.Errorf("failed to write cid field t.Info: %w", err) } - // t.t.DePledgedCollateral (types.BigInt) + // t.t.DePledgedCollateral (types.BigInt) (struct) if err := t.DePledgedCollateral.MarshalCBOR(w); err != nil { return err } - // t.t.DePledgeTime (types.BigInt) + // t.t.DePledgeTime (types.BigInt) (struct) if err := t.DePledgeTime.MarshalCBOR(w); err != nil { return err } - // t.t.Sectors (cid.Cid) + // t.t.Sectors (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Sectors); err != nil { return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) } - // t.t.ProvingSet (cid.Cid) + // t.t.ProvingSet (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProvingSet); err != nil { return xerrors.Errorf("failed to write cid field t.ProvingSet: %w", err) } - // t.t.CurrentFaultSet (types.BitField) + // t.t.CurrentFaultSet (types.BitField) (struct) if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { return err } - // t.t.NextFaultSet (types.BitField) + // t.t.NextFaultSet (types.BitField) (struct) if err := t.NextFaultSet.MarshalCBOR(w); err != nil { return err } - // t.t.NextDoneSet (types.BitField) + // t.t.NextDoneSet (types.BitField) (struct) if err := t.NextDoneSet.MarshalCBOR(w); err != nil { return err } - // t.t.Power (types.BigInt) + // t.t.Power (types.BigInt) (struct) if err := t.Power.MarshalCBOR(w); err != nil { return err } - // t.t.SlashedAt (types.BigInt) + // t.t.SlashedAt (types.BigInt) (struct) if err := t.SlashedAt.MarshalCBOR(w); err != nil { return err } - // t.t.OwedStorageCollateral (types.BigInt) + // t.t.OwedStorageCollateral (types.BigInt) (struct) if err := t.OwedStorageCollateral.MarshalCBOR(w); err != nil { return err } - // t.t.ProvingPeriodEnd (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProvingPeriodEnd)); err != nil { + // t.t.ProvingPeriodEnd (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProvingPeriodEnd))); err != nil { return err } return nil @@ -281,7 +281,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Info (cid.Cid) + // t.t.Info (cid.Cid) (struct) { @@ -293,7 +293,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.Info = c } - // t.t.DePledgedCollateral (types.BigInt) + // t.t.DePledgedCollateral (types.BigInt) (struct) { @@ -302,7 +302,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.DePledgeTime (types.BigInt) + // t.t.DePledgeTime (types.BigInt) (struct) { @@ -311,7 +311,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Sectors (cid.Cid) + // t.t.Sectors (cid.Cid) (struct) { @@ -323,7 +323,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.Sectors = c } - // t.t.ProvingSet (cid.Cid) + // t.t.ProvingSet (cid.Cid) (struct) { @@ -335,7 +335,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.ProvingSet = c } - // t.t.CurrentFaultSet (types.BitField) + // t.t.CurrentFaultSet (types.BitField) (struct) { @@ -344,7 +344,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.NextFaultSet (types.BitField) + // t.t.NextFaultSet (types.BitField) (struct) { @@ -353,7 +353,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.NextDoneSet (types.BitField) + // t.t.NextDoneSet (types.BitField) (struct) { @@ -362,7 +362,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Power (types.BigInt) + // t.t.Power (types.BigInt) (struct) { @@ -371,7 +371,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SlashedAt (types.BigInt) + // t.t.SlashedAt (types.BigInt) (struct) { @@ -380,7 +380,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.OwedStorageCollateral (types.BigInt) + // t.t.OwedStorageCollateral (types.BigInt) (struct) { @@ -389,7 +389,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProvingPeriodEnd (uint64) + // t.t.ProvingPeriodEnd (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -398,7 +398,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProvingPeriodEnd = extra + t.ProvingPeriodEnd = uint64(extra) return nil } @@ -411,22 +411,22 @@ func (t *StorageMinerConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -451,7 +451,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -460,7 +460,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -469,7 +469,7 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -478,8 +478,8 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra - // t.t.PeerID (peer.ID) + t.SectorSize = uint64(extra) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -501,7 +501,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.CommD ([]uint8) + // t.t.CommD ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { return err } @@ -509,7 +509,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.CommR ([]uint8) + // t.t.CommR ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { return err } @@ -517,12 +517,12 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Epoch (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Epoch)); err != nil { + // t.t.Epoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Epoch))); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -530,7 +530,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -540,8 +540,8 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.SectorNumber (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorNumber)); err != nil { + // t.t.SectorNumber (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { return err } return nil @@ -562,7 +562,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.CommD ([]uint8) + // t.t.CommD ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -579,7 +579,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommD); err != nil { return err } - // t.t.CommR ([]uint8) + // t.t.CommR ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -596,7 +596,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } - // t.t.Epoch (uint64) + // t.t.Epoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -605,8 +605,8 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Epoch = extra - // t.t.Proof ([]uint8) + t.Epoch = uint64(extra) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -623,7 +623,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Proof); err != nil { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -653,7 +653,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { t.DealIDs[i] = val } - // t.t.SectorNumber (uint64) + // t.t.SectorNumber (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -662,7 +662,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorNumber = extra + t.SectorNumber = uint64(extra) return nil } @@ -675,17 +675,17 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -693,8 +693,8 @@ func (t *MinerInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } return nil @@ -715,7 +715,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -724,7 +724,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -733,7 +733,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -743,7 +743,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { t.PeerID = peer.ID(sval) } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -752,7 +752,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra + t.SectorSize = uint64(extra) return nil } @@ -765,7 +765,7 @@ func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -773,7 +773,7 @@ func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.DoneSet (types.BitField) + // t.t.DoneSet (types.BitField) (struct) if err := t.DoneSet.MarshalCBOR(w); err != nil { return err } @@ -795,7 +795,7 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -812,7 +812,7 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Proof); err != nil { return err } - // t.t.DoneSet (types.BitField) + // t.t.DoneSet (types.BitField) (struct) { @@ -833,7 +833,7 @@ func (t *PaymentVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Extra ([]uint8) + // t.t.Extra ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Extra)))); err != nil { return err } @@ -841,7 +841,7 @@ func (t *PaymentVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -866,7 +866,7 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Extra ([]uint8) + // t.t.Extra ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -883,7 +883,7 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Extra); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -912,7 +912,7 @@ func (t *UpdatePeerIDParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -937,7 +937,7 @@ func (t *UpdatePeerIDParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -959,7 +959,7 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Signers)))); err != nil { return err } @@ -969,32 +969,32 @@ func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { } } - // t.t.Required (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Required)); err != nil { + // t.t.Required (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Required))); err != nil { return err } - // t.t.NextTxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextTxID)); err != nil { + // t.t.NextTxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextTxID))); err != nil { return err } - // t.t.InitialBalance (types.BigInt) + // t.t.InitialBalance (types.BigInt) (struct) if err := t.InitialBalance.MarshalCBOR(w); err != nil { return err } - // t.t.StartingBlock (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.StartingBlock)); err != nil { + // t.t.StartingBlock (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.StartingBlock))); err != nil { return err } - // t.t.UnlockDuration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + // t.t.UnlockDuration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.UnlockDuration))); err != nil { return err } - // t.t.Transactions ([]actors.MTransaction) + // t.t.Transactions ([]actors.MTransaction) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Transactions)))); err != nil { return err } @@ -1021,7 +1021,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1047,7 +1047,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { t.Signers[i] = v } - // t.t.Required (uint64) + // t.t.Required (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1056,8 +1056,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Required = extra - // t.t.NextTxID (uint64) + t.Required = uint64(extra) + // t.t.NextTxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1066,8 +1066,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextTxID = extra - // t.t.InitialBalance (types.BigInt) + t.NextTxID = uint64(extra) + // t.t.InitialBalance (types.BigInt) (struct) { @@ -1076,7 +1076,7 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.StartingBlock (uint64) + // t.t.StartingBlock (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1085,8 +1085,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.StartingBlock = extra - // t.t.UnlockDuration (uint64) + t.StartingBlock = uint64(extra) + // t.t.UnlockDuration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1095,8 +1095,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.UnlockDuration = extra - // t.t.Transactions ([]actors.MTransaction) + t.UnlockDuration = uint64(extra) + // t.t.Transactions ([]actors.MTransaction) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1134,7 +1134,7 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Signers)))); err != nil { return err } @@ -1144,13 +1144,13 @@ func (t *MultiSigConstructorParams) MarshalCBOR(w io.Writer) error { } } - // t.t.Required (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Required)); err != nil { + // t.t.Required (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Required))); err != nil { return err } - // t.t.UnlockDuration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.UnlockDuration)); err != nil { + // t.t.UnlockDuration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.UnlockDuration))); err != nil { return err } return nil @@ -1171,7 +1171,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signers ([]address.Address) + // t.t.Signers ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1197,7 +1197,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { t.Signers[i] = v } - // t.t.Required (uint64) + // t.t.Required (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1206,8 +1206,8 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Required = extra - // t.t.UnlockDuration (uint64) + t.Required = uint64(extra) + // t.t.UnlockDuration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1216,7 +1216,7 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.UnlockDuration = extra + t.UnlockDuration = uint64(extra) return nil } @@ -1229,22 +1229,22 @@ func (t *MultiSigProposeParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -1269,7 +1269,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1278,7 +1278,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) { @@ -1287,7 +1287,7 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1296,8 +1296,8 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1326,8 +1326,8 @@ func (t *MultiSigTxID) MarshalCBOR(w io.Writer) error { return err } - // t.t.TxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TxID)); err != nil { + // t.t.TxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TxID))); err != nil { return err } return nil @@ -1348,7 +1348,7 @@ func (t *MultiSigTxID) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.TxID (uint64) + // t.t.TxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1357,7 +1357,7 @@ func (t *MultiSigTxID) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TxID = extra + t.TxID = uint64(extra) return nil } @@ -1370,12 +1370,12 @@ func (t *MultiSigSwapSignerParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } @@ -1397,7 +1397,7 @@ func (t *MultiSigSwapSignerParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -1406,7 +1406,7 @@ func (t *MultiSigSwapSignerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1427,8 +1427,8 @@ func (t *MultiSigChangeReqParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Req (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Req)); err != nil { + // t.t.Req (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Req))); err != nil { return err } return nil @@ -1449,7 +1449,7 @@ func (t *MultiSigChangeReqParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Req (uint64) + // t.t.Req (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1458,7 +1458,7 @@ func (t *MultiSigChangeReqParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Req = extra + t.Req = uint64(extra) return nil } @@ -1471,32 +1471,32 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { return err } - // t.t.Created (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Created)); err != nil { + // t.t.Created (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Created))); err != nil { return err } - // t.t.TxID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TxID)); err != nil { + // t.t.TxID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TxID))); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -1504,7 +1504,7 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { return err } - // t.t.Approved ([]address.Address) + // t.t.Approved ([]address.Address) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Approved)))); err != nil { return err } @@ -1514,18 +1514,18 @@ func (t *MTransaction) MarshalCBOR(w io.Writer) error { } } - // t.t.Complete (bool) + // t.t.Complete (bool) (bool) if err := cbg.WriteBool(w, t.Complete); err != nil { return err } - // t.t.Canceled (bool) + // t.t.Canceled (bool) (bool) if err := cbg.WriteBool(w, t.Canceled); err != nil { return err } - // t.t.RetCode (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.RetCode)); err != nil { + // t.t.RetCode (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RetCode))); err != nil { return err } return nil @@ -1546,7 +1546,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Created (uint64) + // t.t.Created (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1555,8 +1555,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Created = extra - // t.t.TxID (uint64) + t.Created = uint64(extra) + // t.t.TxID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1565,8 +1565,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TxID = extra - // t.t.To (address.Address) + t.TxID = uint64(extra) + // t.t.To (address.Address) (struct) { @@ -1575,7 +1575,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) { @@ -1584,7 +1584,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1593,8 +1593,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1611,7 +1611,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Params); err != nil { return err } - // t.t.Approved ([]address.Address) + // t.t.Approved ([]address.Address) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1637,7 +1637,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { t.Approved[i] = v } - // t.t.Complete (bool) + // t.t.Complete (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1654,7 +1654,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.Canceled (bool) + // t.t.Canceled (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1671,7 +1671,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.RetCode (uint64) + // t.t.RetCode (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1680,7 +1680,7 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.RetCode = extra + t.RetCode = uint64(extra) return nil } @@ -1693,12 +1693,12 @@ func (t *MultiSigRemoveSignerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) if err := t.Signer.MarshalCBOR(w); err != nil { return err } - // t.t.Decrease (bool) + // t.t.Decrease (bool) (bool) if err := cbg.WriteBool(w, t.Decrease); err != nil { return err } @@ -1720,7 +1720,7 @@ func (t *MultiSigRemoveSignerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) { @@ -1729,7 +1729,7 @@ func (t *MultiSigRemoveSignerParam) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Decrease (bool) + // t.t.Decrease (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1758,12 +1758,12 @@ func (t *MultiSigAddSignerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) if err := t.Signer.MarshalCBOR(w); err != nil { return err } - // t.t.Increase (bool) + // t.t.Increase (bool) (bool) if err := cbg.WriteBool(w, t.Increase); err != nil { return err } @@ -1785,7 +1785,7 @@ func (t *MultiSigAddSignerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Signer (address.Address) + // t.t.Signer (address.Address) (struct) { @@ -1794,7 +1794,7 @@ func (t *MultiSigAddSignerParam) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Increase (bool) + // t.t.Increase (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1823,32 +1823,32 @@ func (t *PaymentChannelActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.ToSend (types.BigInt) + // t.t.ToSend (types.BigInt) (struct) if err := t.ToSend.MarshalCBOR(w); err != nil { return err } - // t.t.ClosingAt (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ClosingAt)); err != nil { + // t.t.ClosingAt (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ClosingAt))); err != nil { return err } - // t.t.MinCloseHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinCloseHeight)); err != nil { + // t.t.MinCloseHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinCloseHeight))); err != nil { return err } - // t.t.LaneStates (map[string]*actors.LaneState) + // t.t.LaneStates (map[string]*actors.LaneState) (map) if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil { return err } @@ -1885,7 +1885,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -1894,7 +1894,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -1903,7 +1903,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ToSend (types.BigInt) + // t.t.ToSend (types.BigInt) (struct) { @@ -1912,7 +1912,7 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ClosingAt (uint64) + // t.t.ClosingAt (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1921,8 +1921,8 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ClosingAt = extra - // t.t.MinCloseHeight (uint64) + t.ClosingAt = uint64(extra) + // t.t.MinCloseHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1931,8 +1931,8 @@ func (t *PaymentChannelActorState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinCloseHeight = extra - // t.t.LaneStates (map[string]*actors.LaneState) + t.MinCloseHeight = uint64(extra) + // t.t.LaneStates (map[string]*actors.LaneState) (map) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1997,7 +1997,7 @@ func (t *PCAConstructorParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } @@ -2019,7 +2019,7 @@ func (t *PCAConstructorParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -2040,18 +2040,18 @@ func (t *LaneState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Closed (bool) + // t.t.Closed (bool) (bool) if err := cbg.WriteBool(w, t.Closed); err != nil { return err } - // t.t.Redeemed (types.BigInt) + // t.t.Redeemed (types.BigInt) (struct) if err := t.Redeemed.MarshalCBOR(w); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } return nil @@ -2072,7 +2072,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Closed (bool) + // t.t.Closed (bool) (bool) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2089,7 +2089,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { default: return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) } - // t.t.Redeemed (types.BigInt) + // t.t.Redeemed (types.BigInt) (struct) { @@ -2098,7 +2098,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2107,7 +2107,7 @@ func (t *LaneState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra + t.Nonce = uint64(extra) return nil } @@ -2120,12 +2120,12 @@ func (t *PCAUpdateChannelStateParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Sv (types.SignedVoucher) + // t.t.Sv (types.SignedVoucher) (struct) if err := t.Sv.MarshalCBOR(w); err != nil { return err } - // t.t.Secret ([]uint8) + // t.t.Secret ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Secret)))); err != nil { return err } @@ -2133,7 +2133,7 @@ func (t *PCAUpdateChannelStateParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -2158,7 +2158,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Sv (types.SignedVoucher) + // t.t.Sv (types.SignedVoucher) (struct) { @@ -2167,7 +2167,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Secret ([]uint8) + // t.t.Secret ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2184,7 +2184,7 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Secret); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2213,17 +2213,17 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.PayChActor (address.Address) + // t.t.PayChActor (address.Address) (struct) if err := t.PayChActor.MarshalCBOR(w); err != nil { return err } - // t.t.Payer (address.Address) + // t.t.Payer (address.Address) (struct) if err := t.Payer.MarshalCBOR(w); err != nil { return err } - // t.t.ChannelMessage (cid.Cid) + // t.t.ChannelMessage (cid.Cid) (struct) if t.ChannelMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -2235,7 +2235,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.Vouchers ([]*types.SignedVoucher) + // t.t.Vouchers ([]*types.SignedVoucher) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { return err } @@ -2262,7 +2262,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PayChActor (address.Address) + // t.t.PayChActor (address.Address) (struct) { @@ -2271,7 +2271,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Payer (address.Address) + // t.t.Payer (address.Address) (struct) { @@ -2280,7 +2280,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ChannelMessage (cid.Cid) + // t.t.ChannelMessage (cid.Cid) (struct) { @@ -2304,7 +2304,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Vouchers ([]*types.SignedVoucher) + // t.t.Vouchers ([]*types.SignedVoucher) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2342,18 +2342,18 @@ func (t *StoragePowerState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miners (cid.Cid) + // t.t.Miners (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Miners); err != nil { return xerrors.Errorf("failed to write cid field t.Miners: %w", err) } - // t.t.MinerCount (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinerCount)); err != nil { + // t.t.MinerCount (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinerCount))); err != nil { return err } - // t.t.TotalStorage (types.BigInt) + // t.t.TotalStorage (types.BigInt) (struct) if err := t.TotalStorage.MarshalCBOR(w); err != nil { return err } @@ -2375,7 +2375,7 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miners (cid.Cid) + // t.t.Miners (cid.Cid) (struct) { @@ -2387,7 +2387,7 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { t.Miners = c } - // t.t.MinerCount (uint64) + // t.t.MinerCount (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2396,8 +2396,8 @@ func (t *StoragePowerState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinerCount = extra - // t.t.TotalStorage (types.BigInt) + t.MinerCount = uint64(extra) + // t.t.TotalStorage (types.BigInt) (struct) { @@ -2418,22 +2418,22 @@ func (t *CreateStorageMinerParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) if err := t.Owner.MarshalCBOR(w); err != nil { return err } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) if err := t.Worker.MarshalCBOR(w); err != nil { return err } - // t.t.SectorSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorSize)); err != nil { + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { return err } - // t.t.PeerID (peer.ID) + // t.t.PeerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.PeerID)))); err != nil { return err } @@ -2458,7 +2458,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Owner (address.Address) + // t.t.Owner (address.Address) (struct) { @@ -2467,7 +2467,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Worker (address.Address) + // t.t.Worker (address.Address) (struct) { @@ -2476,7 +2476,7 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.SectorSize (uint64) + // t.t.SectorSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2485,8 +2485,8 @@ func (t *CreateStorageMinerParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorSize = extra - // t.t.PeerID (peer.ID) + t.SectorSize = uint64(extra) + // t.t.PeerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -2508,7 +2508,7 @@ func (t *IsMinerParam) MarshalCBOR(w io.Writer) error { return err } - // t.t.Addr (address.Address) + // t.t.Addr (address.Address) (struct) if err := t.Addr.MarshalCBOR(w); err != nil { return err } @@ -2530,7 +2530,7 @@ func (t *IsMinerParam) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Addr (address.Address) + // t.t.Addr (address.Address) (struct) { @@ -2551,7 +2551,7 @@ func (t *PowerLookupParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } @@ -2573,7 +2573,7 @@ func (t *PowerLookupParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -2594,7 +2594,7 @@ func (t *UpdateStorageParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Delta (types.BigInt) + // t.t.Delta (types.BigInt) (struct) if err := t.Delta.MarshalCBOR(w); err != nil { return err } @@ -2616,7 +2616,7 @@ func (t *UpdateStorageParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Delta (types.BigInt) + // t.t.Delta (types.BigInt) (struct) { @@ -2637,12 +2637,12 @@ func (t *ArbitrateConsensusFaultParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Block1 (types.BlockHeader) + // t.t.Block1 (types.BlockHeader) (struct) if err := t.Block1.MarshalCBOR(w); err != nil { return err } - // t.t.Block2 (types.BlockHeader) + // t.t.Block2 (types.BlockHeader) (struct) if err := t.Block2.MarshalCBOR(w); err != nil { return err } @@ -2664,7 +2664,7 @@ func (t *ArbitrateConsensusFaultParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Block1 (types.BlockHeader) + // t.t.Block1 (types.BlockHeader) (struct) { @@ -2685,7 +2685,7 @@ func (t *ArbitrateConsensusFaultParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Block2 (types.BlockHeader) + // t.t.Block2 (types.BlockHeader) (struct) { @@ -2718,7 +2718,7 @@ func (t *PledgeCollateralParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Size (types.BigInt) + // t.t.Size (types.BigInt) (struct) if err := t.Size.MarshalCBOR(w); err != nil { return err } @@ -2740,7 +2740,7 @@ func (t *PledgeCollateralParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Size (types.BigInt) + // t.t.Size (types.BigInt) (struct) { @@ -2761,17 +2761,17 @@ func (t *MinerSlashConsensusFault) MarshalCBOR(w io.Writer) error { return err } - // t.t.Slasher (address.Address) + // t.t.Slasher (address.Address) (struct) if err := t.Slasher.MarshalCBOR(w); err != nil { return err } - // t.t.AtHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.AtHeight)); err != nil { + // t.t.AtHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.AtHeight))); err != nil { return err } - // t.t.SlashedCollateral (types.BigInt) + // t.t.SlashedCollateral (types.BigInt) (struct) if err := t.SlashedCollateral.MarshalCBOR(w); err != nil { return err } @@ -2793,7 +2793,7 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Slasher (address.Address) + // t.t.Slasher (address.Address) (struct) { @@ -2802,7 +2802,7 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { } } - // t.t.AtHeight (uint64) + // t.t.AtHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2811,8 +2811,8 @@ func (t *MinerSlashConsensusFault) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.AtHeight = extra - // t.t.SlashedCollateral (types.BigInt) + t.AtHeight = uint64(extra) + // t.t.SlashedCollateral (types.BigInt) (struct) { @@ -2833,12 +2833,12 @@ func (t *StorageParticipantBalance) MarshalCBOR(w io.Writer) error { return err } - // t.t.Locked (types.BigInt) + // t.t.Locked (types.BigInt) (struct) if err := t.Locked.MarshalCBOR(w); err != nil { return err } - // t.t.Available (types.BigInt) + // t.t.Available (types.BigInt) (struct) if err := t.Available.MarshalCBOR(w); err != nil { return err } @@ -2860,7 +2860,7 @@ func (t *StorageParticipantBalance) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Locked (types.BigInt) + // t.t.Locked (types.BigInt) (struct) { @@ -2869,7 +2869,7 @@ func (t *StorageParticipantBalance) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Available (types.BigInt) + // t.t.Available (types.BigInt) (struct) { @@ -2890,20 +2890,20 @@ func (t *StorageMarketState) MarshalCBOR(w io.Writer) error { return err } - // t.t.Balances (cid.Cid) + // t.t.Balances (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Balances); err != nil { return xerrors.Errorf("failed to write cid field t.Balances: %w", err) } - // t.t.Deals (cid.Cid) + // t.t.Deals (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Deals); err != nil { return xerrors.Errorf("failed to write cid field t.Deals: %w", err) } - // t.t.NextDealID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextDealID)); err != nil { + // t.t.NextDealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextDealID))); err != nil { return err } return nil @@ -2924,7 +2924,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Balances (cid.Cid) + // t.t.Balances (cid.Cid) (struct) { @@ -2936,7 +2936,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { t.Balances = c } - // t.t.Deals (cid.Cid) + // t.t.Deals (cid.Cid) (struct) { @@ -2948,7 +2948,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { t.Deals = c } - // t.t.NextDealID (uint64) + // t.t.NextDealID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -2957,7 +2957,7 @@ func (t *StorageMarketState) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextDealID = extra + t.NextDealID = uint64(extra) return nil } @@ -2970,7 +2970,7 @@ func (t *WithdrawBalanceParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) if err := t.Balance.MarshalCBOR(w); err != nil { return err } @@ -2992,7 +2992,7 @@ func (t *WithdrawBalanceParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) { @@ -3013,7 +3013,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceRef ([]uint8) + // t.t.PieceRef ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { return err } @@ -3021,47 +3021,47 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSize)); err != nil { + // t.t.PieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { return err } - // t.t.PieceSerialization (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.PieceSerialization)); err != nil { + // t.t.PieceSerialization (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { return err } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err } - // t.t.Provider (address.Address) + // t.t.Provider (address.Address) (struct) if err := t.Provider.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalExpiration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + // t.t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { return err } - // t.t.Duration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + // t.t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { return err } - // t.t.StoragePricePerEpoch (types.BigInt) + // t.t.StoragePricePerEpoch (types.BigInt) (struct) if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { return err } - // t.t.StorageCollateral (types.BigInt) + // t.t.StorageCollateral (types.BigInt) (struct) if err := t.StorageCollateral.MarshalCBOR(w); err != nil { return err } - // t.t.ProposerSignature (types.Signature) + // t.t.ProposerSignature (types.Signature) (struct) if err := t.ProposerSignature.MarshalCBOR(w); err != nil { return err } @@ -3083,7 +3083,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PieceRef ([]uint8) + // t.t.PieceRef ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3100,7 +3100,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.PieceRef); err != nil { return err } - // t.t.PieceSize (uint64) + // t.t.PieceSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3109,8 +3109,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.PieceSize = extra - // t.t.PieceSerialization (uint64) + t.PieceSize = uint64(extra) + // t.t.PieceSerialization (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3119,8 +3119,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.PieceSerialization = extra - // t.t.Client (address.Address) + t.PieceSerialization = uint64(extra) + // t.t.Client (address.Address) (struct) { @@ -3129,7 +3129,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Provider (address.Address) + // t.t.Provider (address.Address) (struct) { @@ -3138,7 +3138,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalExpiration (uint64) + // t.t.ProposalExpiration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3147,8 +3147,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProposalExpiration = extra - // t.t.Duration (uint64) + t.ProposalExpiration = uint64(extra) + // t.t.Duration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3157,8 +3157,8 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Duration = extra - // t.t.StoragePricePerEpoch (types.BigInt) + t.Duration = uint64(extra) + // t.t.StoragePricePerEpoch (types.BigInt) (struct) { @@ -3167,7 +3167,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.StorageCollateral (types.BigInt) + // t.t.StorageCollateral (types.BigInt) (struct) { @@ -3176,7 +3176,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposerSignature (types.Signature) + // t.t.ProposerSignature (types.Signature) (struct) { @@ -3209,12 +3209,12 @@ func (t *StorageDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.CounterSignature (types.Signature) + // t.t.CounterSignature (types.Signature) (struct) if err := t.CounterSignature.MarshalCBOR(w); err != nil { return err } @@ -3236,7 +3236,7 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -3245,7 +3245,7 @@ func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.CounterSignature (types.Signature) + // t.t.CounterSignature (types.Signature) (struct) { @@ -3278,7 +3278,7 @@ func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deals ([]actors.StorageDeal) + // t.t.Deals ([]actors.StorageDeal) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3305,7 +3305,7 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deals ([]actors.StorageDeal) + // t.t.Deals ([]actors.StorageDeal) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3343,7 +3343,7 @@ func (t *PublishStorageDealResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -3370,7 +3370,7 @@ func (t *PublishStorageDealResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3412,7 +3412,7 @@ func (t *ActivateStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deals ([]uint64) + // t.t.Deals ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3439,7 +3439,7 @@ func (t *ActivateStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deals ([]uint64) + // t.t.Deals ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3481,7 +3481,7 @@ func (t *ProcessStorageDealsPaymentParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } @@ -3508,7 +3508,7 @@ func (t *ProcessStorageDealsPaymentParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealIDs ([]uint64) + // t.t.DealIDs ([]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3550,13 +3550,13 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deal (actors.StorageDeal) + // t.t.Deal (actors.StorageDeal) (struct) if err := t.Deal.MarshalCBOR(w); err != nil { return err } - // t.t.ActivationEpoch (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ActivationEpoch)); err != nil { + // t.t.ActivationEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ActivationEpoch))); err != nil { return err } return nil @@ -3577,7 +3577,7 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deal (actors.StorageDeal) + // t.t.Deal (actors.StorageDeal) (struct) { @@ -3586,7 +3586,7 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ActivationEpoch (uint64) + // t.t.ActivationEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3595,6 +3595,6 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ActivationEpoch = extra + t.ActivationEpoch = uint64(extra) return nil } diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 40413170c..ec46764f9 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -24,7 +24,7 @@ func (t *AskRequest) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } @@ -46,7 +46,7 @@ func (t *AskRequest) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -67,7 +67,7 @@ func (t *AskResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.Ask (types.SignedStorageAsk) + // t.t.Ask (types.SignedStorageAsk) (struct) if err := t.Ask.MarshalCBOR(w); err != nil { return err } @@ -89,7 +89,7 @@ func (t *AskResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Ask (types.SignedStorageAsk) + // t.t.Ask (types.SignedStorageAsk) (struct) { @@ -122,7 +122,7 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.DealProposal (actors.StorageDealProposal) + // t.t.DealProposal (actors.StorageDealProposal) (struct) if err := t.DealProposal.MarshalCBOR(w); err != nil { return err } @@ -144,7 +144,7 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.DealProposal (actors.StorageDealProposal) + // t.t.DealProposal (actors.StorageDealProposal) (struct) { @@ -165,12 +165,12 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return err } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Message (string) + // t.t.Message (string) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { return err } @@ -178,18 +178,18 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (cid.Cid) + // t.t.Proposal (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Proposal); err != nil { return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) } - // t.t.StorageDeal (actors.StorageDeal) + // t.t.StorageDeal (actors.StorageDeal) (struct) if err := t.StorageDeal.MarshalCBOR(w); err != nil { return err } - // t.t.PublishMessage (cid.Cid) + // t.t.PublishMessage (cid.Cid) (struct) if t.PublishMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -201,7 +201,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { } } - // t.t.CommitMessage (cid.Cid) + // t.t.CommitMessage (cid.Cid) (struct) if t.CommitMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -231,7 +231,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -240,8 +240,8 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Message (string) + t.State = uint64(extra) + // t.t.Message (string) (string) { sval, err := cbg.ReadString(br) @@ -251,7 +251,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Message = string(sval) } - // t.t.Proposal (cid.Cid) + // t.t.Proposal (cid.Cid) (struct) { @@ -263,7 +263,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Proposal = c } - // t.t.StorageDeal (actors.StorageDeal) + // t.t.StorageDeal (actors.StorageDeal) (struct) { @@ -284,7 +284,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { } } - // t.t.PublishMessage (cid.Cid) + // t.t.PublishMessage (cid.Cid) (struct) { @@ -308,7 +308,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { } } - // t.t.CommitMessage (cid.Cid) + // t.t.CommitMessage (cid.Cid) (struct) { @@ -344,12 +344,12 @@ func (t *SignedResponse) MarshalCBOR(w io.Writer) error { return err } - // t.t.Response (deals.Response) + // t.t.Response (deals.Response) (struct) if err := t.Response.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -371,7 +371,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Response (deals.Response) + // t.t.Response (deals.Response) (struct) { @@ -380,7 +380,7 @@ func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -413,38 +413,38 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Data (cid.Cid) + // t.t.Data (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Data); err != nil { return xerrors.Errorf("failed to write cid field t.Data: %w", err) } - // t.t.PricePerEpoch (types.BigInt) + // t.t.PricePerEpoch (types.BigInt) (struct) if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalExpiration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.ProposalExpiration)); err != nil { + // t.t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { return err } - // t.t.Duration (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Duration)); err != nil { + // t.t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { return err } - // t.t.ProviderAddress (address.Address) + // t.t.ProviderAddress (address.Address) (struct) if err := t.ProviderAddress.MarshalCBOR(w); err != nil { return err } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err } - // t.t.MinerID (peer.ID) + // t.t.MinerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { return err } @@ -469,7 +469,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Data (cid.Cid) + // t.t.Data (cid.Cid) (struct) { @@ -481,7 +481,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { t.Data = c } - // t.t.PricePerEpoch (types.BigInt) + // t.t.PricePerEpoch (types.BigInt) (struct) { @@ -490,7 +490,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalExpiration (uint64) + // t.t.ProposalExpiration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -499,8 +499,8 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.ProposalExpiration = extra - // t.t.Duration (uint64) + t.ProposalExpiration = uint64(extra) + // t.t.Duration (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -509,8 +509,8 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Duration = extra - // t.t.ProviderAddress (address.Address) + t.Duration = uint64(extra) + // t.t.ProviderAddress (address.Address) (struct) { @@ -519,7 +519,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Client (address.Address) + // t.t.Client (address.Address) (struct) { @@ -528,7 +528,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinerID (peer.ID) + // t.t.MinerID (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -550,23 +550,23 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Miner (peer.ID) + // t.t.Miner (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil { return err } @@ -591,7 +591,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) { @@ -603,7 +603,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { t.ProposalCid = c } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -612,7 +612,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -621,8 +621,8 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Miner (peer.ID) + t.State = uint64(extra) + // t.t.Miner (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -644,7 +644,7 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Client (peer.ID) + // t.t.Client (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil { return err } @@ -652,35 +652,35 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) if err := t.Proposal.MarshalCBOR(w); err != nil { return err } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ProposalCid); err != nil { return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err) } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.Ref (cid.Cid) + // t.t.Ref (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Ref); err != nil { return xerrors.Errorf("failed to write cid field t.Ref: %w", err) } - // t.t.DealID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.DealID)); err != nil { + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err } - // t.t.SectorID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { return err } return nil @@ -701,7 +701,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Client (peer.ID) + // t.t.Client (peer.ID) (string) { sval, err := cbg.ReadString(br) @@ -711,7 +711,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.Client = peer.ID(sval) } - // t.t.Proposal (actors.StorageDealProposal) + // t.t.Proposal (actors.StorageDealProposal) (struct) { @@ -720,7 +720,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { } } - // t.t.ProposalCid (cid.Cid) + // t.t.ProposalCid (cid.Cid) (struct) { @@ -732,7 +732,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.ProposalCid = c } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -741,8 +741,8 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.Ref (cid.Cid) + t.State = uint64(extra) + // t.t.Ref (cid.Cid) (struct) { @@ -754,7 +754,7 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.Ref = c } - // t.t.DealID (uint64) + // t.t.DealID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -763,8 +763,8 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.DealID = extra - // t.t.SectorID (uint64) + t.DealID = uint64(extra) + // t.t.SectorID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -773,6 +773,6 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorID = extra + t.SectorID = uint64(extra) return nil } diff --git a/chain/deals/state_store.go b/chain/deals/state_store.go index ae959146f..cde6240be 100644 --- a/chain/deals/state_store.go +++ b/chain/deals/state_store.go @@ -2,6 +2,7 @@ package deals import ( "bytes" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index ab63b28a1..a1a6adf35 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -23,12 +23,12 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } - // t.t.Tickets ([]*types.Ticket) + // t.t.Tickets ([]*types.Ticket) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Tickets)))); err != nil { return err } @@ -38,7 +38,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } } - // t.t.ElectionProof ([]uint8) + // t.t.ElectionProof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.ElectionProof)))); err != nil { return err } @@ -46,7 +46,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return err } - // t.t.Parents ([]cid.Cid) + // t.t.Parents ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Parents)))); err != nil { return err } @@ -56,45 +56,45 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } } - // t.t.ParentWeight (types.BigInt) + // t.t.ParentWeight (types.BigInt) (struct) if err := t.ParentWeight.MarshalCBOR(w); err != nil { return err } - // t.t.Height (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Height)); err != nil { + // t.t.Height (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Height))); err != nil { return err } - // t.t.ParentStateRoot (cid.Cid) + // t.t.ParentStateRoot (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ParentStateRoot); err != nil { return xerrors.Errorf("failed to write cid field t.ParentStateRoot: %w", err) } - // t.t.ParentMessageReceipts (cid.Cid) + // t.t.ParentMessageReceipts (cid.Cid) (struct) if err := cbg.WriteCid(w, t.ParentMessageReceipts); err != nil { return xerrors.Errorf("failed to write cid field t.ParentMessageReceipts: %w", err) } - // t.t.Messages (cid.Cid) + // t.t.Messages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Messages); err != nil { return xerrors.Errorf("failed to write cid field t.Messages: %w", err) } - // t.t.BLSAggregate (types.Signature) + // t.t.BLSAggregate (types.Signature) (struct) if err := t.BLSAggregate.MarshalCBOR(w); err != nil { return err } - // t.t.Timestamp (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Timestamp)); err != nil { + // t.t.Timestamp (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { return err } - // t.t.BlockSig (types.Signature) + // t.t.BlockSig (types.Signature) (struct) if err := t.BlockSig.MarshalCBOR(w); err != nil { return err } @@ -116,7 +116,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) { @@ -125,7 +125,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Tickets ([]*types.Ticket) + // t.t.Tickets ([]*types.Ticket) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -151,7 +151,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Tickets[i] = &v } - // t.t.ElectionProof ([]uint8) + // t.t.ElectionProof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -168,7 +168,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.ElectionProof); err != nil { return err } - // t.t.Parents ([]cid.Cid) + // t.t.Parents ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -193,7 +193,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Parents[i] = c } - // t.t.ParentWeight (types.BigInt) + // t.t.ParentWeight (types.BigInt) (struct) { @@ -202,7 +202,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Height (uint64) + // t.t.Height (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -211,8 +211,8 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Height = extra - // t.t.ParentStateRoot (cid.Cid) + t.Height = uint64(extra) + // t.t.ParentStateRoot (cid.Cid) (struct) { @@ -224,7 +224,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.ParentStateRoot = c } - // t.t.ParentMessageReceipts (cid.Cid) + // t.t.ParentMessageReceipts (cid.Cid) (struct) { @@ -236,7 +236,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.ParentMessageReceipts = c } - // t.t.Messages (cid.Cid) + // t.t.Messages (cid.Cid) (struct) { @@ -248,7 +248,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { t.Messages = c } - // t.t.BLSAggregate (types.Signature) + // t.t.BLSAggregate (types.Signature) (struct) { @@ -257,7 +257,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Timestamp (uint64) + // t.t.Timestamp (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -266,8 +266,8 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Timestamp = extra - // t.t.BlockSig (types.Signature) + t.Timestamp = uint64(extra) + // t.t.BlockSig (types.Signature) (struct) { @@ -288,7 +288,7 @@ func (t *Ticket) MarshalCBOR(w io.Writer) error { return err } - // t.t.VRFProof ([]uint8) + // t.t.VRFProof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.VRFProof)))); err != nil { return err } @@ -313,7 +313,7 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.VRFProof ([]uint8) + // t.t.VRFProof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -342,42 +342,42 @@ func (t *Message) MarshalCBOR(w io.Writer) error { return err } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) if err := t.To.MarshalCBOR(w); err != nil { return err } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) if err := t.From.MarshalCBOR(w); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Value (types.BigInt) + // t.t.Value (types.BigInt) (struct) if err := t.Value.MarshalCBOR(w); err != nil { return err } - // t.t.GasPrice (types.BigInt) + // t.t.GasPrice (types.BigInt) (struct) if err := t.GasPrice.MarshalCBOR(w); err != nil { return err } - // t.t.GasLimit (types.BigInt) + // t.t.GasLimit (types.BigInt) (struct) if err := t.GasLimit.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Params ([]uint8) + // t.t.Params ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Params)))); err != nil { return err } @@ -402,7 +402,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.To (address.Address) + // t.t.To (address.Address) (struct) { @@ -411,7 +411,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.From (address.Address) + // t.t.From (address.Address) (struct) { @@ -420,7 +420,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -429,8 +429,8 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Value (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Value (types.BigInt) (struct) { @@ -439,7 +439,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.GasPrice (types.BigInt) + // t.t.GasPrice (types.BigInt) (struct) { @@ -448,7 +448,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.GasLimit (types.BigInt) + // t.t.GasLimit (types.BigInt) (struct) { @@ -457,7 +457,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -466,8 +466,8 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Params ([]uint8) + t.Method = uint64(extra) + // t.t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -496,12 +496,12 @@ func (t *SignedMessage) MarshalCBOR(w io.Writer) error { return err } - // t.t.Message (types.Message) + // t.t.Message (types.Message) (struct) if err := t.Message.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -523,7 +523,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Message (types.Message) + // t.t.Message (types.Message) (struct) { @@ -532,7 +532,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -553,13 +553,13 @@ func (t *MsgMeta) MarshalCBOR(w io.Writer) error { return err } - // t.t.BlsMessages (cid.Cid) + // t.t.BlsMessages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.BlsMessages); err != nil { return xerrors.Errorf("failed to write cid field t.BlsMessages: %w", err) } - // t.t.SecpkMessages (cid.Cid) + // t.t.SecpkMessages (cid.Cid) (struct) if err := cbg.WriteCid(w, t.SecpkMessages); err != nil { return xerrors.Errorf("failed to write cid field t.SecpkMessages: %w", err) @@ -583,7 +583,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.BlsMessages (cid.Cid) + // t.t.BlsMessages (cid.Cid) (struct) { @@ -595,7 +595,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { t.BlsMessages = c } - // t.t.SecpkMessages (cid.Cid) + // t.t.SecpkMessages (cid.Cid) (struct) { @@ -619,12 +619,12 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { return err } - // t.t.TimeLock (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.TimeLock)); err != nil { + // t.t.TimeLock (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TimeLock))); err != nil { return err } - // t.t.SecretPreimage ([]uint8) + // t.t.SecretPreimage ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.SecretPreimage)))); err != nil { return err } @@ -632,32 +632,32 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { return err } - // t.t.Extra (types.ModVerifyParams) + // t.t.Extra (types.ModVerifyParams) (struct) if err := t.Extra.MarshalCBOR(w); err != nil { return err } - // t.t.Lane (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Lane)); err != nil { + // t.t.Lane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Amount (types.BigInt) + // t.t.Amount (types.BigInt) (struct) if err := t.Amount.MarshalCBOR(w); err != nil { return err } - // t.t.MinCloseHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinCloseHeight)); err != nil { + // t.t.MinCloseHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinCloseHeight))); err != nil { return err } - // t.t.Merges ([]types.Merge) + // t.t.Merges ([]types.Merge) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Merges)))); err != nil { return err } @@ -667,7 +667,7 @@ func (t *SignedVoucher) MarshalCBOR(w io.Writer) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -689,7 +689,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.TimeLock (uint64) + // t.t.TimeLock (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -698,8 +698,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TimeLock = extra - // t.t.SecretPreimage ([]uint8) + t.TimeLock = uint64(extra) + // t.t.SecretPreimage ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -716,7 +716,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.SecretPreimage); err != nil { return err } - // t.t.Extra (types.ModVerifyParams) + // t.t.Extra (types.ModVerifyParams) (struct) { @@ -737,7 +737,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Lane (uint64) + // t.t.Lane (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -746,8 +746,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Lane = extra - // t.t.Nonce (uint64) + t.Lane = uint64(extra) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -756,8 +756,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Amount (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Amount (types.BigInt) (struct) { @@ -766,7 +766,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinCloseHeight (uint64) + // t.t.MinCloseHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -775,8 +775,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinCloseHeight = extra - // t.t.Merges ([]types.Merge) + t.MinCloseHeight = uint64(extra) + // t.t.Merges ([]types.Merge) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -802,7 +802,7 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { t.Merges[i] = v } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -835,17 +835,17 @@ func (t *ModVerifyParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Actor (address.Address) + // t.t.Actor (address.Address) (struct) if err := t.Actor.MarshalCBOR(w); err != nil { return err } - // t.t.Method (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Method)); err != nil { + // t.t.Method (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } - // t.t.Data ([]uint8) + // t.t.Data ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { return err } @@ -870,7 +870,7 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Actor (address.Address) + // t.t.Actor (address.Address) (struct) { @@ -879,7 +879,7 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Method (uint64) + // t.t.Method (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -888,8 +888,8 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Method = extra - // t.t.Data ([]uint8) + t.Method = uint64(extra) + // t.t.Data ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -918,13 +918,13 @@ func (t *Merge) MarshalCBOR(w io.Writer) error { return err } - // t.t.Lane (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Lane)); err != nil { + // t.t.Lane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Lane))); err != nil { return err } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } return nil @@ -945,7 +945,7 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Lane (uint64) + // t.t.Lane (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -954,8 +954,8 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Lane = extra - // t.t.Nonce (uint64) + t.Lane = uint64(extra) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -964,7 +964,7 @@ func (t *Merge) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra + t.Nonce = uint64(extra) return nil } @@ -977,24 +977,24 @@ func (t *Actor) MarshalCBOR(w io.Writer) error { return err } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Code); err != nil { return xerrors.Errorf("failed to write cid field t.Code: %w", err) } - // t.t.Head (cid.Cid) + // t.t.Head (cid.Cid) (struct) if err := cbg.WriteCid(w, t.Head); err != nil { return xerrors.Errorf("failed to write cid field t.Head: %w", err) } - // t.t.Nonce (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Nonce)); err != nil { + // t.t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } - // t.t.Balance (types.BigInt) + // t.t.Balance (types.BigInt) (struct) if err := t.Balance.MarshalCBOR(w); err != nil { return err } @@ -1016,7 +1016,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Code (cid.Cid) + // t.t.Code (cid.Cid) (struct) { @@ -1028,7 +1028,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { t.Code = c } - // t.t.Head (cid.Cid) + // t.t.Head (cid.Cid) (struct) { @@ -1040,7 +1040,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { t.Head = c } - // t.t.Nonce (uint64) + // t.t.Nonce (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1049,8 +1049,8 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Nonce = extra - // t.t.Balance (types.BigInt) + t.Nonce = uint64(extra) + // t.t.Balance (types.BigInt) (struct) { @@ -1071,12 +1071,12 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.t.ExitCode (uint8) + // t.t.ExitCode (uint8) (uint8) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ExitCode))); err != nil { return err } - // t.t.Return ([]uint8) + // t.t.Return ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Return)))); err != nil { return err } @@ -1084,7 +1084,7 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.t.GasUsed (types.BigInt) + // t.t.GasUsed (types.BigInt) (struct) if err := t.GasUsed.MarshalCBOR(w); err != nil { return err } @@ -1106,7 +1106,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.ExitCode (uint8) + // t.t.ExitCode (uint8) (uint8) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1119,7 +1119,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("integer in input was too large for uint8 field") } t.ExitCode = uint8(extra) - // t.t.Return ([]uint8) + // t.t.Return ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1136,7 +1136,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.Return); err != nil { return err } - // t.t.GasUsed (types.BigInt) + // t.t.GasUsed (types.BigInt) (struct) { @@ -1157,12 +1157,12 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { return err } - // t.t.Header (types.BlockHeader) + // t.t.Header (types.BlockHeader) (struct) if err := t.Header.MarshalCBOR(w); err != nil { return err } - // t.t.BlsMessages ([]cid.Cid) + // t.t.BlsMessages ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.BlsMessages)))); err != nil { return err } @@ -1172,7 +1172,7 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { } } - // t.t.SecpkMessages ([]cid.Cid) + // t.t.SecpkMessages ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.SecpkMessages)))); err != nil { return err } @@ -1199,7 +1199,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Header (types.BlockHeader) + // t.t.Header (types.BlockHeader) (struct) { @@ -1220,7 +1220,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { } } - // t.t.BlsMessages ([]cid.Cid) + // t.t.BlsMessages ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1245,7 +1245,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { t.BlsMessages[i] = c } - // t.t.SecpkMessages ([]cid.Cid) + // t.t.SecpkMessages ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1282,12 +1282,12 @@ func (t *SignedStorageAsk) MarshalCBOR(w io.Writer) error { return err } - // t.t.Ask (types.StorageAsk) + // t.t.Ask (types.StorageAsk) (struct) if err := t.Ask.MarshalCBOR(w); err != nil { return err } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) if err := t.Signature.MarshalCBOR(w); err != nil { return err } @@ -1309,7 +1309,7 @@ func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Ask (types.StorageAsk) + // t.t.Ask (types.StorageAsk) (struct) { @@ -1330,7 +1330,7 @@ func (t *SignedStorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Signature (types.Signature) + // t.t.Signature (types.Signature) (struct) { @@ -1363,33 +1363,33 @@ func (t *StorageAsk) MarshalCBOR(w io.Writer) error { return err } - // t.t.Price (types.BigInt) + // t.t.Price (types.BigInt) (struct) if err := t.Price.MarshalCBOR(w); err != nil { return err } - // t.t.MinPieceSize (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.MinPieceSize)); err != nil { + // t.t.MinPieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.MinPieceSize))); err != nil { return err } - // t.t.Miner (address.Address) + // t.t.Miner (address.Address) (struct) if err := t.Miner.MarshalCBOR(w); err != nil { return err } - // t.t.Timestamp (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Timestamp)); err != nil { + // t.t.Timestamp (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { return err } - // t.t.Expiry (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Expiry)); err != nil { + // t.t.Expiry (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Expiry))); err != nil { return err } - // t.t.SeqNo (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SeqNo)); err != nil { + // t.t.SeqNo (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SeqNo))); err != nil { return err } return nil @@ -1410,7 +1410,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Price (types.BigInt) + // t.t.Price (types.BigInt) (struct) { @@ -1419,7 +1419,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.MinPieceSize (uint64) + // t.t.MinPieceSize (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1428,8 +1428,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.MinPieceSize = extra - // t.t.Miner (address.Address) + t.MinPieceSize = uint64(extra) + // t.t.Miner (address.Address) (struct) { @@ -1438,7 +1438,7 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Timestamp (uint64) + // t.t.Timestamp (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1447,8 +1447,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Timestamp = extra - // t.t.Expiry (uint64) + t.Timestamp = uint64(extra) + // t.t.Expiry (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1457,8 +1457,8 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Expiry = extra - // t.t.SeqNo (uint64) + t.Expiry = uint64(extra) + // t.t.SeqNo (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1467,6 +1467,6 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SeqNo = extra + t.SeqNo = uint64(extra) return nil } diff --git a/chain/types/voucher.go b/chain/types/voucher.go index 7855b21a2..1cb68406c 100644 --- a/chain/types/voucher.go +++ b/chain/types/voucher.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "github.com/filecoin-project/lotus/chain/address" + cborrpc "github.com/filecoin-project/lotus/lib/cborrpc" cbor "github.com/ipfs/go-ipld-cbor" ) @@ -46,13 +47,13 @@ func (sv *SignedVoucher) EncodedString() (string, error) { func (sv *SignedVoucher) Equals(other *SignedVoucher) bool { // TODO: make this less bad - selfB, err := cbor.DumpObject(sv) + selfB, err := cborrpc.Dump(sv) if err != nil { log.Errorf("SignedVoucher.Equals: dump self: %s", err) return false } - otherB, err := cbor.DumpObject(other) + otherB, err := cborrpc.Dump(other) if err != nil { log.Errorf("SignedVoucher.Equals: dump other: %s", err) return false diff --git a/gen/main.go b/gen/main.go index f7b4c68ef..f274e4430 100644 --- a/gen/main.go +++ b/gen/main.go @@ -6,10 +6,12 @@ import ( gen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" + "github.com/filecoin-project/lotus/retrieval" ) func main() { @@ -42,6 +44,29 @@ func main() { os.Exit(1) } + err = gen.WriteTupleEncodersToFile("./api/cbor_gen.go", "api", + api.PaymentInfo{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval", + retrieval.RetParams{}, + + retrieval.Query{}, + retrieval.QueryResponse{}, + retrieval.Unixfs0Offer{}, + retrieval.DealProposal{}, + retrieval.DealResponse{}, + retrieval.Block{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + /* err = gen.WriteTupleEncodersToFile("./chain/cbor_gen.go", "chain", chain.BlockSyncRequest{}, diff --git a/go.mod b/go.mod index 4ad2e0779..da117bec0 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c + github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.0 @@ -84,7 +84,7 @@ require ( go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect - golang.org/x/time v0.0.0-20181108054448-85acf8d2951c + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 google.golang.org/api v0.9.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 diff --git a/go.sum b/go.sum index 518e5e1eb..28d6ded71 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16/go.mod h1:lKjJYPg2kwbav5f78i5YA8kGccnZn18IySbpneXvaQs= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug= @@ -166,6 +167,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -178,6 +180,7 @@ github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -187,6 +190,7 @@ github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= @@ -196,12 +200,15 @@ github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMi github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-blockservice v0.0.1/go.mod h1:2Ao89U7jV1KIqqNk5EdhSTBG/Pgc1vMFr0bhkx376j4= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 h1:qYLz/x/d1SOiiFGS8dwCBCFJ5Oh64Y8HMBrS+MbaU8c= github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4/go.mod h1:NSSM0pxlhej9rSFXQmB/lDru7TYNoDjKgmvqzlsd06Y= +github.com/ipfs/go-car v0.0.1 h1:Nn3RjJbysnDud4wILAybzOAvzsaycrCoKM4BMIK0Y04= +github.com/ipfs/go-car v0.0.1/go.mod h1:pUz3tUIpudsTch0ZQrEPOvNUBT1LufCXg8aZ4KOrEMM= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= @@ -242,14 +249,17 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -259,6 +269,7 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144 h1:5WM8S1nwquWQ3zEuNhK82NE5Di6Pd41qz9JxxvxTAIA= github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144/go.mod h1:azGN5dH7ailfREknDDNYB0Eq4qZ/4I4Y3gO0ivjJNyM= +github.com/ipfs/go-merkledag v0.0.1/go.mod h1:CRdtHMROECqaehAGeJ0Wd9TtlmWv/ta5cUnvbTnniEI= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -275,6 +286,7 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -321,6 +333,8 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= +github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= @@ -328,15 +342,19 @@ github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvM github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-libp2p v0.0.1/go.mod h1:bmRs8I0vwn6iRaVssZnJx/epY6WPSKiLoK1vyle4EX0= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p v0.3.0 h1:XhYEJKmIdi4U4Zbie/ym9k6lqgg3PHM2stGS/cOUDWk= github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= +github.com/libp2p/go-libp2p-autonat v0.0.1/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= +github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= +github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= @@ -350,44 +368,61 @@ github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYe github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= +github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= +github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= +github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= +github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= +github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= github.com/libp2p/go-libp2p-kad-dht v0.1.1/go.mod h1:1kj2Rk5pX3/0RwqMm9AMNCT7DzcMHYhgDN5VTi+cY0M= github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= github.com/libp2p/go-libp2p-kbucket v0.2.0/go.mod h1:JNymBToym3QXKBMKGy3m29+xprg0EVr/GJFHxFEdgh8= +github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= +github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= +github.com/libp2p/go-libp2p-nat v0.0.1/go.mod h1:4L6ajyUIlJvx1Cbh5pc6Ma6vMDpKXf3GgLO5u7W0oQ4= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= +github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= +github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= +github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= +github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= +github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= +github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= +github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= +github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= +github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= @@ -399,17 +434,23 @@ github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/ github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-tls v0.1.0 h1:o4bjjAdnUjNgJoPoDd0wUaZH7K+EenlNWJpgyXB3ulA= github.com/libp2p/go-libp2p-tls v0.1.0/go.mod h1:VZdoSWQDeNpIIAFJFv+6uqTqpnIIDHcqZQSTC/A1TT0= +github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= +github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= +github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= +github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= +github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= +github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= @@ -420,14 +461,18 @@ github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3 github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= +github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= +github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= +github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= +github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -456,6 +501,7 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.16 h1:iMEQ/IVHxPTtx2Q07JP/k4CKRvSjiAZjZ0hnhgYEDmE= github.com/miekg/dns v1.1.16/go.mod h1:YNV562EiewvSmpCB6/W4c6yqjK7Z+M/aIS1JHsIVeg8= @@ -500,6 +546,7 @@ github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKT github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= +github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -628,8 +675,8 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c h1:P3leakVQLSvS0b953QG0BcuoCul79KTNpHbE04Bxhmc= -github.com/whyrusleeping/cbor-gen v0.0.0-20191104184210-9aa3672a465c/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f h1:+GFA37QICd1Axd2n9uzjtvPjxJJI5PU78vpvam+hI4U= +github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -637,6 +684,9 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= +github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= +github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= +github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30 h1:nMCC9Pwz1pxfC1Y6mYncdk+kq8d5aLx0Q+/gyZGE44M= @@ -647,6 +697,7 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d h1:wnjWu1N8UT github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7ckxrjiFh8mi1AY7ox23PZD0g6QU/TxW3U3unX7I3A= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= +github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -693,6 +744,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -755,6 +807,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go new file mode 100644 index 000000000..626eef75d --- /dev/null +++ b/lib/sectorbuilder/mock.go @@ -0,0 +1,42 @@ +package sectorbuilder + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/filecoin-project/lotus/chain/address" +) + +func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + return nil, nil, err + } + + addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") + if err != nil { + return nil, nil, err + } + + metadata := filepath.Join(dir, "meta") + sealed := filepath.Join(dir, "sealed") + staging := filepath.Join(dir, "staging") + + sb, err := New(&SectorBuilderConfig{ + SectorSize: sectorSize, + SealedDir: sealed, + StagedDir: staging, + MetadataDir: metadata, + Miner: addr, + }) + if err != nil { + return nil, nil, err + } + + return sb, func() { + if err := os.RemoveAll(dir); err != nil { + log.Warn("failed to clean up temp sectorbuilder: ", err) + } + }, nil +} diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index b40a0b9cd..903373cc0 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -3,15 +3,12 @@ package sectorbuilder_test import ( "context" "io" - "io/ioutil" "math/rand" - "path/filepath" "testing" "github.com/ipfs/go-datastore" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sector" ) @@ -26,30 +23,11 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - dir, err := ioutil.TempDir("", "sbtest") - if err != nil { - t.Fatal(err) - } - - addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") - if err != nil { - t.Fatal(err) - } - - metadata := filepath.Join(dir, "meta") - sealed := filepath.Join(dir, "sealed") - staging := filepath.Join(dir, "staging") - - sb, err := sectorbuilder.New(§orbuilder.SectorBuilderConfig{ - SectorSize: sectorSize, - SealedDir: sealed, - StagedDir: staging, - MetadataDir: metadata, - Miner: addr, - }) + sb, cleanup, err := TempSectorbuilder(sectorSize) if err != nil { t.Fatal(err) } + defer cleanup() // TODO: Consider fixing store := sector.NewStore(sb, datastore.NewMapDatastore(), func(ctx context.Context) (*sectorbuilder.SealTicket, error) { diff --git a/node/hello/hello.go b/node/hello/hello.go index d512d1d27..ef37fae38 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -3,6 +3,7 @@ package hello import ( "context" "fmt" + "go.uber.org/fx" "github.com/ipfs/go-cid" diff --git a/node/impl/client/client.go b/node/impl/client/client.go index a785bef44..85e9ca4f7 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -3,11 +3,12 @@ package client import ( "context" "errors" - "golang.org/x/xerrors" "io" "math" "os" + "golang.org/x/xerrors" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" diff --git a/paych/cbor_gen.go b/paych/cbor_gen.go index 5eac7c08e..e640f3057 100644 --- a/paych/cbor_gen.go +++ b/paych/cbor_gen.go @@ -22,12 +22,12 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Voucher (types.SignedVoucher) + // t.t.Voucher (types.SignedVoucher) (struct) if err := t.Voucher.MarshalCBOR(w); err != nil { return err } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { return err } @@ -52,7 +52,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Voucher (types.SignedVoucher) + // t.t.Voucher (types.SignedVoucher) (struct) { @@ -73,7 +73,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Proof ([]uint8) + // t.t.Proof ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -102,27 +102,27 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Channel (address.Address) + // t.t.Channel (address.Address) (struct) if err := t.Channel.MarshalCBOR(w); err != nil { return err } - // t.t.Control (address.Address) + // t.t.Control (address.Address) (struct) if err := t.Control.MarshalCBOR(w); err != nil { return err } - // t.t.Target (address.Address) + // t.t.Target (address.Address) (struct) if err := t.Target.MarshalCBOR(w); err != nil { return err } - // t.t.Direction (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Direction)); err != nil { + // t.t.Direction (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Direction))); err != nil { return err } - // t.t.Vouchers ([]*paych.VoucherInfo) + // t.t.Vouchers ([]*paych.VoucherInfo) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil { return err } @@ -132,8 +132,8 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.NextLane (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.NextLane)); err != nil { + // t.t.NextLane (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextLane))); err != nil { return err } return nil @@ -154,7 +154,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Channel (address.Address) + // t.t.Channel (address.Address) (struct) { @@ -163,7 +163,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Control (address.Address) + // t.t.Control (address.Address) (struct) { @@ -172,7 +172,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Target (address.Address) + // t.t.Target (address.Address) (struct) { @@ -181,7 +181,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.Direction (uint64) + // t.t.Direction (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -190,8 +190,8 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Direction = extra - // t.t.Vouchers ([]*paych.VoucherInfo) + t.Direction = uint64(extra) + // t.t.Vouchers ([]*paych.VoucherInfo) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -217,7 +217,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { t.Vouchers[i] = &v } - // t.t.NextLane (uint64) + // t.t.NextLane (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -226,6 +226,6 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.NextLane = extra + t.NextLane = uint64(extra) return nil } diff --git a/retrieval/cbor_gen.go b/retrieval/cbor_gen.go new file mode 100644 index 000000000..93d800697 --- /dev/null +++ b/retrieval/cbor_gen.go @@ -0,0 +1,473 @@ +package retrieval + +import ( + "fmt" + "io" + "io/ioutil" + + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *RetParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + if err := t.Unixfs0.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *RetParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Unixfs0 (retrieval.Unixfs0Offer) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Unixfs0 = new(Unixfs0Offer) + if err := t.Unixfs0.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + return nil +} + +func (t *Query) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Piece (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Piece); err != nil { + return xerrors.Errorf("failed to write cid field t.Piece: %w", err) + } + + return nil +} + +func (t *Query) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Piece (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Piece: %w", err) + } + + t.Piece = c + + } + return nil +} + +func (t *QueryResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Status (retrieval.QueryResponseStatus) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + + // t.t.MinPrice (types.BigInt) (struct) + if err := t.MinPrice.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *QueryResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Status (retrieval.QueryResponseStatus) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = QueryResponseStatus(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + // t.t.MinPrice (types.BigInt) (struct) + + { + + if err := t.MinPrice.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Offset (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + return nil +} + +func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Offset (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + return nil +} + +func (t *DealProposal) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Payment (api.PaymentInfo) (struct) + if err := t.Payment.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Ref (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Ref); err != nil { + return xerrors.Errorf("failed to write cid field t.Ref: %w", err) + } + + // t.t.Params (retrieval.RetParams) (struct) + if err := t.Params.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *DealProposal) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Payment (api.PaymentInfo) (struct) + + { + + if err := t.Payment.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Ref (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Ref: %w", err) + } + + t.Ref = c + + } + // t.t.Params (retrieval.RetParams) (struct) + + { + + if err := t.Params.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + +func (t *DealResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Status (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { + return err + } + + // t.t.Message (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Message)); err != nil { + return err + } + return nil +} + +func (t *DealResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Status (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = uint64(extra) + // t.t.Message (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Message = string(sval) + } + return nil +} + +func (t *Block) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Prefix ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil { + return err + } + if _, err := w.Write(t.Prefix); err != nil { + return err + } + + // t.t.Data ([]uint8) (slice) + dlen := len(t.Data) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(dlen))); err != nil { + return err + } + if n, err := w.Write(t.Data); err != nil { + return err + } else if n != dlen { + return fmt.Errorf("somehow wrote the wrong number of bytes...") + } + return nil +} + +func (t *Block) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + data, err := ioutil.ReadAll(r) + if err != nil { + panic("piss") + } + fmt.Println("STRING DATA: ", string(data)) + return fmt.Errorf("cbor input should be of type array (got %d)", maj) + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Prefix ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Prefix: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Prefix = make([]byte, extra) + if _, err := io.ReadFull(br, t.Prefix); err != nil { + return err + } + // t.t.Data ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Data: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Data = make([]byte, extra) + if _, err := io.ReadFull(br, t.Data); err != nil { + return err + } + return nil +} diff --git a/retrieval/client.go b/retrieval/client.go index ec086ef49..bc2fa2db3 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -3,11 +3,9 @@ package retrieval import ( "context" "io" - "io/ioutil" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" @@ -45,7 +43,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. } defer s.Close() - err = cborrpc.WriteCborRPC(s, Query{ + err = cborrpc.WriteCborRPC(s, &Query{ Piece: data, }) if err != nil { @@ -53,15 +51,8 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } - // TODO: read deadline - rawResp, err := ioutil.ReadAll(s) - if err != nil { - log.Warn(err) - return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} - } - var resp QueryResponse - if err := cbor.DecodeInto(rawResp, &resp); err != nil { + if err := resp.UnmarshalCBOR(s); err != nil { log.Warn(err) return api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID} } @@ -167,7 +158,7 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out return xerrors.Errorf("setting up retrieval payment: %w", err) } - deal := DealProposal{ + deal := &DealProposal{ Payment: payment, Ref: cst.root, Params: RetParams{ diff --git a/retrieval/miner.go b/retrieval/miner.go index 350177fe2..39bb6f417 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -2,6 +2,7 @@ package retrieval import ( "context" + "fmt" "io" "github.com/ipfs/go-blockservice" @@ -37,7 +38,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner { func writeErr(stream network.Stream, err error) { log.Errorf("Retrieval deal error: %s", err) - _ = cborrpc.WriteCborRPC(stream, DealResponse{ + _ = cborrpc.WriteCborRPC(stream, &DealResponse{ Status: Error, Message: err.Error(), }) @@ -58,7 +59,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { return } - answer := QueryResponse{ + answer := &QueryResponse{ Status: Unavailable, } if err == nil { @@ -134,7 +135,7 @@ func (hnd *handlerDeal) handleNext() (bool, error) { // If the file isn't open (new deal stream), isn't the right file, or isn't // at the right offset, (re)open it if hnd.open != deal.Ref || hnd.at != unixfs0.Offset { - log.Infof("opening file for sending (open '%s') (@%d, want %d)", hnd.open, hnd.at, unixfs0.Offset) + log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset) if err := hnd.openFile(deal); err != nil { return false, err } @@ -195,7 +196,7 @@ func (hnd *handlerDeal) openFile(deal DealProposal) error { func (hnd *handlerDeal) accept(deal DealProposal) error { unixfs0 := deal.Params.Unixfs0 - resp := DealResponse{ + resp := &DealResponse{ Status: Accepted, } if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil { @@ -203,6 +204,8 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { return err } + defer fmt.Println("leaving accept retrieval deal") + blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize for i := uint64(0); i < blocksToSend; { data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) @@ -221,11 +224,12 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { return }*/ - block := Block{ + block := &Block{ Prefix: nd.Cid().Prefix().Bytes(), Data: nd.RawData(), } + fmt.Println("retrieval sending block: ", i, blocksToSend, len(nd.RawData())) if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { return err } diff --git a/retrieval/types.go b/retrieval/types.go index a7a66ea51..d82d8b138 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -3,7 +3,6 @@ package retrieval import ( "github.com/filecoin-project/lotus/api" "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" "github.com/filecoin-project/lotus/chain/types" ) @@ -11,7 +10,7 @@ import ( const ProtocolID = "/fil/retrieval/-1.0.0" // TODO: spec const QueryProtocolID = "/fil/retrieval/qry/-1.0.0" // TODO: spec -type QueryResponseStatus int +type QueryResponseStatus uint64 const ( Available QueryResponseStatus = iota @@ -26,15 +25,6 @@ const ( ) func init() { - cbor.RegisterCborType(RetParams{}) - - cbor.RegisterCborType(Query{}) - cbor.RegisterCborType(QueryResponse{}) - cbor.RegisterCborType(Unixfs0Offer{}) - - cbor.RegisterCborType(DealProposal{}) - cbor.RegisterCborType(DealResponse{}) - cbor.RegisterCborType(Block{}) } type Query struct { @@ -69,7 +59,7 @@ type DealProposal struct { } type DealResponse struct { - Status int + Status uint64 Message string } From 82be4cd77e47fd3e4b501e5bef0a997938a3bc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 14:15:21 +0100 Subject: [PATCH 124/230] Update go-sectorbuilder again --- extern/go-sectorbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index e95a86c6d..2886db51e 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit e95a86c6dac7376dc78550f253ed002455553997 +Subproject commit 2886db51e1b49d697d7ee87ea662e4545225acea From eb28c45c00844d963c8991046ac39f2881abdf6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 15:03:59 +0100 Subject: [PATCH 125/230] events: Plumb context to callbacks --- chain/events/events.go | 6 ++-- chain/events/events_called.go | 3 +- chain/events/events_height.go | 9 ++++-- chain/events/events_test.go | 48 ++++++++++++++-------------- lib/sectorbuilder/sectorbuilder.go | 6 ++-- lotuspond/front/src/chain/methods.js | 9 +++--- storage/post.go | 10 +++--- storage/sector_states.go | 6 ++-- 8 files changed, 51 insertions(+), 46 deletions(-) diff --git a/chain/events/events.go b/chain/events/events.go index b9decb22f..18287dafe 100644 --- a/chain/events/events.go +++ b/chain/events/events.go @@ -18,8 +18,8 @@ import ( var log = logging.Logger("events") // `curH`-`ts.Height` = `confidence` -type HeightHandler func(ts *types.TipSet, curH uint64) error -type RevertHandler func(ts *types.TipSet) error +type HeightHandler func(ctx context.Context, ts *types.TipSet, curH uint64) error +type RevertHandler func(ctx context.Context, ts *types.TipSet) error type heightHandler struct { confidence int @@ -59,6 +59,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events { heightEvents: heightEvents{ tsc: tsc, + ctx: ctx, gcConfidence: uint64(gcConfidence), heightTriggers: map[uint64]*heightHandler{}, @@ -69,6 +70,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events { calledEvents: calledEvents{ cs: api, tsc: tsc, + ctx: ctx, gcConfidence: uint64(gcConfidence), confQueue: map[triggerH]map[msgH][]*queuedEvent{}, diff --git a/chain/events/events_called.go b/chain/events/events_called.go index 98ea04ec8..c51d9bef6 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -56,6 +56,7 @@ type queuedEvent struct { type calledEvents struct { cs eventApi tsc *tipSetCache + ctx context.Context gcConfidence uint64 lk sync.Mutex @@ -114,7 +115,7 @@ func (e *calledEvents) handleReverts(ts *types.TipSet) { trigger := e.triggers[event.trigger] - if err := trigger.revert(ts); err != nil { + if err := trigger.revert(e.ctx, ts); err != nil { log.Errorf("reverting chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, ts.Height(), triggerH, err) } } diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 2805f38e2..751834de3 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -1,6 +1,7 @@ package events import ( + "context" "sync" "github.com/filecoin-project/lotus/chain/types" @@ -17,6 +18,8 @@ type heightEvents struct { htTriggerHeights map[triggerH][]triggerId htHeights map[msgH][]triggerId + + ctx context.Context } func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { @@ -26,7 +29,7 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { revert := func(h uint64, ts *types.TipSet) { for _, tid := range e.htHeights[h] { - err := e.heightTriggers[tid].revert(ts) + err := e.heightTriggers[tid].revert(e.ctx, ts) if err != nil { log.Errorf("reverting chain trigger (@H %d): %s", h, err) } @@ -74,7 +77,7 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { return err } - if err := hnd.handle(incTs, h); err != nil { + if err := hnd.handle(e.ctx, incTs, h); err != nil { log.Errorf("chain trigger (@H %d, called @ %d) failed: %s", triggerH, ts.Height(), err) } } @@ -125,7 +128,7 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence } e.lk.Unlock() - if err := hnd(ts, bestH); err != nil { + if err := hnd(e.ctx, ts, bestH); err != nil { return err } e.lk.Lock() diff --git a/chain/events/events_test.go b/chain/events/events_test.go index f672bc5f6..d738edf86 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -183,12 +183,12 @@ func TestAt(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -248,12 +248,12 @@ func TestAtNullTrigger(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, uint64(6), ts.Height()) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -282,12 +282,12 @@ func TestAtNullConf(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -323,12 +323,12 @@ func TestAtStart(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 8, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -357,12 +357,12 @@ func TestAtStartConfidence(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 5, int(ts.Height())) require.Equal(t, 11, int(curH)) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -385,16 +385,16 @@ func TestAtChained(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { - return events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { + return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 10) - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -421,16 +421,16 @@ func TestAtChainedConfidence(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { - return events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { + return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { require.Equal(t, 10, int(ts.Height())) applied = true return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 10) - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -455,11 +455,11 @@ func TestAtChainedConfidenceNull(t *testing.T) { var applied bool var reverted bool - err := events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error { applied = true require.Equal(t, 6, int(ts.Height())) return nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 5) @@ -499,7 +499,7 @@ func TestCalled(t *testing.T) { appliedTs = ts appliedH = curH return more, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { reverted = true return nil }, 3, 20, t0123, 5) @@ -693,7 +693,7 @@ func TestCalledTimeout(t *testing.T) { require.Equal(t, uint64(20), ts.Height()) require.Equal(t, uint64(23), curH) return false, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { t.Fatal("revert on timeout") return nil }, 3, 20, t0123, 5) @@ -728,7 +728,7 @@ func TestCalledTimeout(t *testing.T) { require.Equal(t, uint64(20), ts.Height()) require.Equal(t, uint64(23), curH) return false, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { t.Fatal("revert on timeout") return nil }, 3, 20, t0123, 5) @@ -774,7 +774,7 @@ func TestCalledOrder(t *testing.T) { } at++ return true, nil - }, func(ts *types.TipSet) error { + }, func(_ context.Context, ts *types.TipSet) error { switch at { case 2: require.Equal(t, uint64(4), ts.Height()) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 2e6c67ba6..6aebc892b 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -58,7 +58,7 @@ type Config struct { func New(cfg *Config) (*SectorBuilder, error) { if cfg.WorkerThreads <= PoStReservedWorkers { - return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers + 1, cfg.WorkerThreads) + return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads) } proverId := addressToProverID(cfg.Miner) @@ -69,8 +69,8 @@ func New(cfg *Config) (*SectorBuilder, error) { } return &SectorBuilder{ - handle: sbp, - rateLimit: make(chan struct{}, cfg.WorkerThreads - PoStReservedWorkers), + handle: sbp, + rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), }, nil } diff --git a/lotuspond/front/src/chain/methods.js b/lotuspond/front/src/chain/methods.js index b9e153ef7..172531c5a 100644 --- a/lotuspond/front/src/chain/methods.js +++ b/lotuspond/front/src/chain/methods.js @@ -34,8 +34,9 @@ export default { [code.miner]: [ "Send", "Constructor", - "CommitSector", - "SubmitPost", + "PreCommitSector", + "ProveCommitSector", + "SubmitPoSt", "SlashStorageFault", "GetCurrentProvingSet", "ArbitrateDeal", @@ -49,8 +50,8 @@ export default { "ChangeWorker", "IsSlashed", "IsLate", - "PaymentVerifyInclusion", - "PaymentVerifySector", + "DeclareFaults", + "SlashConsensusFault", ], [code.multisig]: [ "Send", diff --git a/storage/post.go b/storage/post.go index 9d3f892a7..4d3e8de1a 100644 --- a/storage/post.go +++ b/storage/post.go @@ -45,7 +45,7 @@ func (m *Miner) beginPosting(ctx context.Context) { m.postLk.Unlock() log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime) - err = m.events.ChainAt(m.computePost(m.schedPost), func(ts *types.TipSet) error { // Revert + err = m.events.ChainAt(m.computePost(m.schedPost), func(ctx context.Context, ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil @@ -84,7 +84,7 @@ func (m *Miner) scheduleNextPost(ppe uint64) { log.Infow("scheduling PoSt", "post-height", ppe-build.PoStChallangeTime, "height", ts.Height(), "ppe", ppe, "proving-period", provingPeriod) - err = m.events.ChainAt(m.computePost(ppe), func(ts *types.TipSet) error { // Revert + err = m.events.ChainAt(m.computePost(ppe), func(ctx context.Context, ts *types.TipSet) error { // Revert // TODO: Cancel post log.Errorf("TODO: Cancel PoSt, re-run") return nil @@ -96,9 +96,9 @@ func (m *Miner) scheduleNextPost(ppe uint64) { } } -func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) error { +func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipSet, curH uint64) error { called := 0 - return func(ts *types.TipSet, curH uint64) error { + return func(ctx context.Context, ts *types.TipSet, curH uint64) error { called++ if called > 1 { log.Errorw("BUG: computePost callback called again", "ppe", ppe, @@ -106,8 +106,6 @@ func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) erro return nil } - ctx := context.TODO() - sset, err := m.api.StateMinerProvingSet(ctx, m.maddr, ts) if err != nil { return xerrors.Errorf("failed to get proving set for miner: %w", err) diff --git a/storage/sector_states.go b/storage/sector_states.go index cbdc68a26..1122ea9a6 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -98,9 +98,9 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect } randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - 1 // -1 because of how the messages are applied - log.Infof("precommit for sector %d made it on chain, will start post computation at height %d", sector.SectorID, randHeight) + log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight) - err = m.events.ChainAt(func(ts *types.TipSet, curH uint64) error { + err = m.events.ChainAt(func(ctx context.Context, ts *types.TipSet, curH uint64) error { m.sectorUpdated <- sectorUpdate{ newState: api.Committing, id: sector.SectorID, @@ -111,7 +111,7 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect } return nil - }, func(ts *types.TipSet) error { + }, func(ctx context.Context, ts *types.TipSet) error { log.Warn("revert in interactive commit sector step") return nil }, 3, mw.TipSet.Height()+build.InteractivePoRepDelay) From 4c4c2095a69d900293732fe21b95dc1f52a311b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 15:36:44 +0100 Subject: [PATCH 126/230] events: Basic tracing --- chain/events/events_height.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 751834de3..3c03f7571 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -2,6 +2,7 @@ package events import ( "context" + "go.opencensus.io/trace" "sync" "github.com/filecoin-project/lotus/chain/types" @@ -23,13 +24,24 @@ type heightEvents struct { } func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { + ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange") + defer span.End() + span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height()))) + span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev)))) + span.AddAttributes(trace.Int64Attribute("applies", int64(len(app)))) + for _, ts := range rev { // TODO: log error if h below gcconfidence // revert height-based triggers revert := func(h uint64, ts *types.TipSet) { for _, tid := range e.htHeights[h] { - err := e.heightTriggers[tid].revert(e.ctx, ts) + ctx, span := trace.StartSpan(ctx, "events.HeightRevert") + + err := e.heightTriggers[tid].revert(ctx, ts) + + span.End() + if err != nil { log.Errorf("reverting chain trigger (@H %d): %s", h, err) } @@ -77,7 +89,13 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error { return err } - if err := hnd.handle(e.ctx, incTs, h); err != nil { + ctx, span := trace.StartSpan(ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", false)) + + err = hnd.handle(ctx, incTs, h) + span.End() + + if err != nil { log.Errorf("chain trigger (@H %d, called @ %d) failed: %s", triggerH, ts.Height(), err) } } @@ -128,9 +146,16 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence } e.lk.Unlock() - if err := hnd(e.ctx, ts, bestH); err != nil { + ctx, span := trace.StartSpan(e.ctx, "events.HeightApply") + span.AddAttributes(trace.BoolAttribute("immediate", true)) + + err = hnd(ctx, ts, bestH) + span.End() + + if err != nil { return err } + e.lk.Lock() bestH = e.tsc.best().Height() } From daabe470a44122867fd6928d3f590054b048f838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 15:52:33 +0100 Subject: [PATCH 127/230] storageminer: tracing in computePost --- storage/post.go | 200 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 141 insertions(+), 59 deletions(-) diff --git a/storage/post.go b/storage/post.go index 4d3e8de1a..3fe45d24c 100644 --- a/storage/post.go +++ b/storage/post.go @@ -2,6 +2,8 @@ package storage import ( "context" + "github.com/ipfs/go-cid" + "go.opencensus.io/trace" "time" "golang.org/x/xerrors" @@ -96,6 +98,140 @@ func (m *Miner) scheduleNextPost(ppe uint64) { } } +type post struct { + m *Miner + + ppe uint64 + ts *types.TipSet + + // prep + sset []*api.SectorInfo + r []byte + + // run + proof []byte + + // commit + smsg cid.Cid +} + +func (p *post) doPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.computePost") + defer span.End() + + if err := p.preparePost(ctx); err != nil { + return err + } + + if err := p.runPost(ctx); err != nil { + return err + } + + if err := p.commitPost(ctx); err != nil { + return err + } + + if err := p.waitCommit(ctx); err != nil { + return err + } + + return nil +} + +func (p *post) preparePost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.preparePost") + defer span.End() + + sset, err := p.m.api.StateMinerProvingSet(ctx, p.m.maddr, p.ts) + if err != nil { + return xerrors.Errorf("failed to get proving set for miner: %w", err) + } + p.sset = sset + + r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, int(int64(p.ts.Height())-int64(p.ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math + if err != nil { + return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err) + } + p.r = r + + return nil +} + +func (p *post) runPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.runPost") + defer span.End() + + log.Infow("running PoSt", "delayed-by", + int64(p.ts.Height())-(int64(p.ppe)-int64(build.PoStChallangeTime)), + "chain-random", p.r, "ppe", p.ppe, "height", p.ts.Height()) + + tsStart := time.Now() + var faults []uint64 // TODO + proof, err := p.m.secst.RunPoSt(ctx, p.sset, p.r, faults) + if err != nil { + return xerrors.Errorf("running post failed: %w", err) + } + elapsed := time.Since(tsStart) + + p.proof = proof + log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed) + + return nil +} + +func (p *post) commitPost(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.commitPost") + defer span.End() + + params := &actors.SubmitPoStParams{ + Proof: p.proof, + DoneSet: types.BitFieldFromSet(nil), + } + + enc, aerr := actors.SerializeParams(params) + if aerr != nil { + return xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + } + + msg := &types.Message{ + To: p.m.maddr, + From: p.m.worker, + Method: actors.MAMethods.SubmitPoSt, + Params: enc, + Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + log.Info("mpush") + + smsg, err := p.m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return xerrors.Errorf("pushing message to mpool: %w", err) + } + p.smsg = smsg.Cid() + + return nil +} + +func (p *post) waitCommit(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "storage.waitPost") + defer span.End() + + log.Infof("Waiting for post %s to appear on chain", p.smsg) + + // make sure it succeeds... + rec, err := p.m.api.StateWaitMsg(ctx, p.smsg) + if err != nil { + return err + } + if rec.Receipt.ExitCode != 0 { + log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) + // TODO: Do something + } + return nil +} + func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipSet, curH uint64) error { called := 0 return func(ctx context.Context, ts *types.TipSet, curH uint64) error { @@ -106,68 +242,14 @@ func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipS return nil } - sset, err := m.api.StateMinerProvingSet(ctx, m.maddr, ts) - if err != nil { - return xerrors.Errorf("failed to get proving set for miner: %w", err) - } - - r, err := m.api.ChainGetRandomness(ctx, ts, nil, int(int64(ts.Height())-int64(ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math - if err != nil { - return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", ts.Height(), ppe, err) - } - - log.Infow("running PoSt", "delayed-by", - int64(ts.Height())-(int64(ppe)-int64(build.PoStChallangeTime)), - "chain-random", r, "ppe", ppe, "height", ts.Height()) - - tsStart := time.Now() - var faults []uint64 - proof, err := m.secst.RunPoSt(ctx, sset, r, faults) - if err != nil { - return xerrors.Errorf("running post failed: %w", err) - } - elapsed := time.Since(tsStart) - - log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed) - - params := &actors.SubmitPoStParams{ - Proof: proof, - DoneSet: types.BitFieldFromSet(nil), - } - - enc, aerr := actors.SerializeParams(params) - if aerr != nil { - return xerrors.Errorf("could not serialize submit post parameters: %w", err) - } - - msg := &types.Message{ - To: m.maddr, - From: m.worker, - Method: actors.MAMethods.SubmitPoSt, - Params: enc, - Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late - GasLimit: types.NewInt(1000000 /* i dont know help */), - GasPrice: types.NewInt(1), - } - - log.Info("mpush") - - smsg, err := m.api.MpoolPushMessage(ctx, msg) - if err != nil { - return xerrors.Errorf("pushing message to mpool: %w", err) - } - - log.Infof("Waiting for post %s to appear on chain", smsg.Cid()) - - // make sure it succeeds... - rec, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + err := (&post{ + m: m, + ppe: ppe, + ts: ts, + }).doPost(ctx) if err != nil { return err } - if rec.Receipt.ExitCode != 0 { - log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) - // TODO: Do something - } m.scheduleNextPost(ppe + build.ProvingPeriodDuration) return nil From 1f56bcf581c84f130acb622b30ee7891b00a7b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 17:15:52 +0100 Subject: [PATCH 128/230] Some param changes --- build/params.go | 5 ++++- chain/actors/actor_miner.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/build/params.go b/build/params.go index ef55c01bd..cd2fc738c 100644 --- a/build/params.go +++ b/build/params.go @@ -55,7 +55,7 @@ const Finality = 500 // Proofs // Blocks -const ProvingPeriodDuration = 60 +const ProvingPeriodDuration = 300 // PoStChallangeTime sets the window in which post computation should happen // Blocks @@ -71,6 +71,9 @@ const PoStRandomnessLookback = 1 // Blocks const SealRandomnessLookback = Finality +// Blocks +const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 + // ///// // Mining diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 2ec36a253..963a469e9 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -231,8 +231,8 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.Epoch, vmctx.BlockHeight()+build.SealRandomnessLookback) } - if vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback > 1000 { - return nil, aerrors.New(2, "sector commitment must be recent enough") + if vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit { + return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback) } mi, err := loadMinerInfo(vmctx, self) From 432125699279fe74ac473fd6ef8c55e4b62f1107 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 08:38:54 -0800 Subject: [PATCH 129/230] fix retrieval protocol error by wrapping stream in peeker --- chain/gen/gen.go | 6 ------ lib/sectorbuilder/sectorbuilder.go | 3 +++ lib/sectorbuilder/sectorbuilder_test.go | 4 ++-- retrieval/cbor_gen.go | 15 +++------------ retrieval/client.go | 7 +++++-- retrieval/miner.go | 7 ++++++- 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index e1452bb91..0d951f1fe 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -279,12 +279,6 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad } fts := store.NewFullTipSet(blks) - fmt.Println("Made a block: ", fts.TipSet().Cids()) - if len(fts.TipSet().Cids()) > 1 { - for _, b := range blks { - fmt.Printf("block %s: %#v\n", b.Cid(), b.Header) - } - } return &MinedTipSet{ TipSet: fts, diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index a2e4499b7..bf55f6927 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -32,6 +32,8 @@ const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer + + Miner address.Address } type SectorBuilderConfig struct { @@ -52,6 +54,7 @@ func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) { return &SectorBuilder{ handle: sbp, + Miner: cfg.Miner, }, nil } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 903373cc0..8122ad9ac 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -23,7 +23,7 @@ func TestSealAndVerify(t *testing.T) { t.Fatal(err) } - sb, cleanup, err := TempSectorbuilder(sectorSize) + sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize) if err != nil { t.Fatal(err) } @@ -53,7 +53,7 @@ func TestSealAndVerify(t *testing.T) { ssinfo := <-store.Incoming() - ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], addr, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) + ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], sb.Miner, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof) if err != nil { t.Fatal(err) } diff --git a/retrieval/cbor_gen.go b/retrieval/cbor_gen.go index 93d800697..3feb15dd1 100644 --- a/retrieval/cbor_gen.go +++ b/retrieval/cbor_gen.go @@ -3,7 +3,6 @@ package retrieval import ( "fmt" "io" - "io/ioutil" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -403,14 +402,11 @@ func (t *Block) MarshalCBOR(w io.Writer) error { } // t.t.Data ([]uint8) (slice) - dlen := len(t.Data) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(dlen))); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil { return err } - if n, err := w.Write(t.Data); err != nil { + if _, err := w.Write(t.Data); err != nil { return err - } else if n != dlen { - return fmt.Errorf("somehow wrote the wrong number of bytes...") } return nil } @@ -423,12 +419,7 @@ func (t *Block) UnmarshalCBOR(r io.Reader) error { return err } if maj != cbg.MajArray { - data, err := ioutil.ReadAll(r) - if err != nil { - panic("piss") - } - fmt.Println("STRING DATA: ", string(data)) - return fmt.Errorf("cbor input should be of type array (got %d)", maj) + return fmt.Errorf("cbor input should be of type array") } if extra != 2 { diff --git a/retrieval/client.go b/retrieval/client.go index bc2fa2db3..23e7daa09 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" @@ -69,6 +70,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. type clientStream struct { payapi payapi.PaychAPI stream network.Stream + peeker cbg.BytePeeker root cid.Cid size types.BigInt @@ -118,6 +120,7 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, cst := clientStream{ payapi: c.payapi, stream: s, + peeker: cbg.GetPeeker(s), root: root, size: types.NewInt(size), @@ -174,7 +177,7 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out } var resp DealResponse - if err := cborrpc.ReadCborRPC(cst.stream, &resp); err != nil { + if err := cborrpc.ReadCborRPC(cst.peeker, &resp); err != nil { log.Error(err) return err } @@ -206,7 +209,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { log.Infof("block %d of %d", i+1, blocksToFetch) var block Block - if err := cborrpc.ReadCborRPC(cst.stream, &block); err != nil { + if err := cborrpc.ReadCborRPC(cst.peeker, &block); err != nil { return xerrors.Errorf("reading fetchBlock response: %w", err) } diff --git a/retrieval/miner.go b/retrieval/miner.go index 39bb6f417..274963a09 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -14,14 +14,19 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/storage/sectorblocks" ) +type RetrMinerApi interface { + PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) +} + type Miner struct { sectorBlocks *sectorblocks.SectorBlocks - full api.FullNode + full RetrMinerApi pricePerByte types.BigInt // TODO: Unseal price From 2816fb886689d1e96540faaa68e9588944ec42be Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 08:56:43 -0800 Subject: [PATCH 130/230] address review feedback --- paych/store.go | 7 ++++--- retrieval/miner.go | 4 ---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/paych/store.go b/paych/store.go index 2b2c1e00d..105b87103 100644 --- a/paych/store.go +++ b/paych/store.go @@ -14,6 +14,7 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + cborrpc "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -59,12 +60,12 @@ func dskeyForChannel(addr address.Address) datastore.Key { func (ps *Store) putChannelInfo(ci *ChannelInfo) error { k := dskeyForChannel(ci.Channel) - buf := new(bytes.Buffer) - if err := ci.MarshalCBOR(buf); err != nil { + b, err := cborrpc.Dump(ci) + if err != nil { return err } - return ps.ds.Put(k, buf.Bytes()) + return ps.ds.Put(k, b) } func (ps *Store) getChannelInfo(addr address.Address) (*ChannelInfo, error) { diff --git a/retrieval/miner.go b/retrieval/miner.go index 274963a09..4a7bd967e 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -2,7 +2,6 @@ package retrieval import ( "context" - "fmt" "io" "github.com/ipfs/go-blockservice" @@ -209,8 +208,6 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { return err } - defer fmt.Println("leaving accept retrieval deal") - blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize for i := uint64(0); i < blocksToSend; { data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO()) @@ -234,7 +231,6 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { Data: nd.RawData(), } - fmt.Println("retrieval sending block: ", i, blocksToSend, len(nd.RawData())) if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { return err } From 6a802cc9dbcdab53d5bf9142665a2f04fe4ce9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 18:09:42 +0100 Subject: [PATCH 131/230] WIP fixing tests --- chain/gen/gen.go | 4 ++-- lib/sectorbuilder/sectorbuilder_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index b38f2c0df..6af8b9d4f 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -279,12 +279,12 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad } fts := store.NewFullTipSet(blks) - fmt.Println("Made a block: ", fts.TipSet().Cids()) + /*fmt.Println("Made a block: ", fts.TipSet().Cids()) if len(fts.TipSet().Cids()) > 1 { for _, b := range blks { fmt.Printf("block %s: %#v\n", b.Cid(), b.Header) } - } + }*/ return &MinedTipSet{ TipSet: fts, diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index ad2d8ebc2..819c52824 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -15,7 +15,9 @@ import ( const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - //t.Skip("this is slow") + t.Skip("this is slow") + //os.Setenv("BELLMAN_NO_GPU", "1") + build.SectorSizes = []uint64{sectorSize} if err := build.GetParams(true); err != nil { @@ -41,7 +43,7 @@ func TestSealAndVerify(t *testing.T) { SectorSize: sectorSize, Miner: addr, - WorkerThreads: 1, + WorkerThreads: 2, CacheDir: cache, SealedDir: sealed, From d79770afb48bac52b89316257859daa3efe25c8b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 09:21:52 -0800 Subject: [PATCH 132/230] Fix nit and go mod tidy --- go.sum | 52 ---------------------------------------------- retrieval/types.go | 3 --- 2 files changed, 55 deletions(-) diff --git a/go.sum b/go.sum index 28d6ded71..88a12b003 100644 --- a/go.sum +++ b/go.sum @@ -78,7 +78,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16/go.mod h1:lKjJYPg2kwbav5f78i5YA8kGccnZn18IySbpneXvaQs= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug= @@ -167,7 +166,6 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= @@ -180,7 +178,6 @@ github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= @@ -190,7 +187,6 @@ github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= github.com/ipfs/go-bitswap v0.1.3/go.mod h1:YEQlFy0kkxops5Vy+OxWdRSEZIoS7I7KDIwoa5Chkps= @@ -200,15 +196,12 @@ github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMi github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= -github.com/ipfs/go-blockservice v0.0.1/go.mod h1:2Ao89U7jV1KIqqNk5EdhSTBG/Pgc1vMFr0bhkx376j4= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4 h1:qYLz/x/d1SOiiFGS8dwCBCFJ5Oh64Y8HMBrS+MbaU8c= github.com/ipfs/go-car v0.0.0-20190823083746-79984a8632b4/go.mod h1:NSSM0pxlhej9rSFXQmB/lDru7TYNoDjKgmvqzlsd06Y= -github.com/ipfs/go-car v0.0.1 h1:Nn3RjJbysnDud4wILAybzOAvzsaycrCoKM4BMIK0Y04= -github.com/ipfs/go-car v0.0.1/go.mod h1:pUz3tUIpudsTch0ZQrEPOvNUBT1LufCXg8aZ4KOrEMM= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= @@ -249,17 +242,14 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= -github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRDI/HQQ= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipld-cbor v0.0.1/go.mod h1:RXHr8s4k0NE0TKhnrxqZC9M888QfsBN9rhS5NjfKzY8= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= @@ -269,7 +259,6 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144 h1:5WM8S1nwquWQ3zEuNhK82NE5Di6Pd41qz9JxxvxTAIA= github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144/go.mod h1:azGN5dH7ailfREknDDNYB0Eq4qZ/4I4Y3gO0ivjJNyM= -github.com/ipfs/go-merkledag v0.0.1/go.mod h1:CRdtHMROECqaehAGeJ0Wd9TtlmWv/ta5cUnvbTnniEI= github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= @@ -286,7 +275,6 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= -github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -333,8 +321,6 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= -github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= -github.com/libp2p/go-conn-security-multistream v0.0.1/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= @@ -342,19 +328,15 @@ github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvM github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= -github.com/libp2p/go-libp2p v0.0.1/go.mod h1:bmRs8I0vwn6iRaVssZnJx/epY6WPSKiLoK1vyle4EX0= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p v0.3.0 h1:XhYEJKmIdi4U4Zbie/ym9k6lqgg3PHM2stGS/cOUDWk= github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0= -github.com/libp2p/go-libp2p-autonat v0.0.1/go.mod h1:fs71q5Xk+pdnKU014o2iq1RhMs9/PMaG5zXRFNnIIT4= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= -github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3 h1:0KycuXvPDhmehw0ASsg+s1o3IfXgCUDqfzAl94KEBOg= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= -github.com/libp2p/go-libp2p-circuit v0.0.1/go.mod h1:Dqm0s/BiV63j8EEAs8hr1H5HudqvCAeXxDyic59lCwE= github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= @@ -368,61 +350,44 @@ github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYe github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= -github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.0.1/go.mod h1:ZkkF9xIFRLA1xCc7bstYFkd80gBGK8Fc1JqGoU2i+zI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= -github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= -github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= -github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= github.com/libp2p/go-libp2p-kad-dht v0.1.1/go.mod h1:1kj2Rk5pX3/0RwqMm9AMNCT7DzcMHYhgDN5VTi+cY0M= github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= github.com/libp2p/go-libp2p-kbucket v0.2.0/go.mod h1:JNymBToym3QXKBMKGy3m29+xprg0EVr/GJFHxFEdgh8= -github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= -github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-nat v0.0.1/go.mod h1:4L6ajyUIlJvx1Cbh5pc6Ma6vMDpKXf3GgLO5u7W0oQ4= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= -github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= -github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= -github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= -github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= -github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= -github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.1.1 h1:ZJK2bHXYUBqObHX+rHLSNrM3M8fmJUlUHrodDPPATmY= github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= -github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ= -github.com/libp2p/go-libp2p-secio v0.0.1/go.mod h1:IdG6iQybdcYmbTzxp4J5dwtUEDTOvZrT0opIDVNPrJs= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= -github.com/libp2p/go-libp2p-swarm v0.0.1/go.mod h1:mh+KZxkbd3lQnveQ3j2q60BM1Cw2mX36XXQqwfPOShs= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= @@ -434,23 +399,17 @@ github.com/libp2p/go-libp2p-testing v0.1.0 h1:WaFRj/t3HdMZGNZqnU2pS7pDRBmMeoDx7/ github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-tls v0.1.0 h1:o4bjjAdnUjNgJoPoDd0wUaZH7K+EenlNWJpgyXB3ulA= github.com/libp2p/go-libp2p-tls v0.1.0/go.mod h1:VZdoSWQDeNpIIAFJFv+6uqTqpnIIDHcqZQSTC/A1TT0= -github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= -github.com/libp2p/go-libp2p-transport v0.0.4/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= -github.com/libp2p/go-libp2p-transport-upgrader v0.0.1/go.mod h1:NJpUAgQab/8K6K0m+JmZCe5RUXG10UMEx4kWe9Ipj5c= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= -github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= -github.com/libp2p/go-mplex v0.0.1/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= -github.com/libp2p/go-msgio v0.0.1/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= @@ -461,18 +420,14 @@ github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3 github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= -github.com/libp2p/go-reuseport-transport v0.0.1/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= -github.com/libp2p/go-tcp-transport v0.0.1/go.mod h1:mnjg0o0O5TmXUaUIanYPUqkW4+u6mK0en8rlpA6BBTs= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= -github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= -github.com/libp2p/go-ws-transport v0.0.1/go.mod h1:p3bKjDWHEgtuKKj+2OdPYs5dAPIjtpQGHF2tJfGz7Ww= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -501,7 +456,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.16 h1:iMEQ/IVHxPTtx2Q07JP/k4CKRvSjiAZjZ0hnhgYEDmE= github.com/miekg/dns v1.1.16/go.mod h1:YNV562EiewvSmpCB6/W4c6yqjK7Z+M/aIS1JHsIVeg8= @@ -546,7 +500,6 @@ github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKT github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.7 h1:uoqoE03rGJdlQEPq2EAc6UeSbo4L7mZyeAAoqNalf54= github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM= -github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -684,9 +637,6 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= -github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4= -github.com/whyrusleeping/go-smux-multistream v2.0.2+incompatible/go.mod h1:dRWHHvc4HDQSHh9gbKEBbUZ+f2Q8iZTPG3UOGYODxSQ= -github.com/whyrusleeping/go-smux-yamux v2.0.8+incompatible/go.mod h1:6qHUzBXUbB9MXmw3AUdB52L8sEb/hScCqOdW2kj/wuI= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30 h1:nMCC9Pwz1pxfC1Y6mYncdk+kq8d5aLx0Q+/gyZGE44M= @@ -697,7 +647,6 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d h1:wnjWu1N8UT github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7ckxrjiFh8mi1AY7ox23PZD0g6QU/TxW3U3unX7I3A= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= -github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -744,7 +693,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/retrieval/types.go b/retrieval/types.go index d82d8b138..28343e6b2 100644 --- a/retrieval/types.go +++ b/retrieval/types.go @@ -24,9 +24,6 @@ const ( Unsealing ) -func init() { -} - type Query struct { Piece cid.Cid // TODO: payment From ebdaa3f0200dc583d6f6e1cfbb09718eb45c1691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 18:33:58 +0100 Subject: [PATCH 133/230] Fix TestSyncBadTimestamp with low BlockDelay --- chain/sync_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chain/sync_test.go b/chain/sync_test.go index 24b8fbd10..d17b6e853 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" @@ -390,7 +391,7 @@ func TestSyncBadTimestamp(t *testing.T) { base := tu.g.CurTipset tu.g.Timestamper = func(pts *types.TipSet, tl int) uint64 { - return pts.MinTimestamp() + 2 + return pts.MinTimestamp() + (build.BlockDelay / 2) } fmt.Println("BASE: ", base.Cids()) From d1b1b6f79f1003d0b308b9da954acfbcf8a9a190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 18:41:04 +0100 Subject: [PATCH 134/230] revert BlockDelay to 10s --- build/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params.go b/build/params.go index cd2fc738c..22e1ece3c 100644 --- a/build/params.go +++ b/build/params.go @@ -37,7 +37,7 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // Consensus / Network // Seconds -const BlockDelay = 2 +const BlockDelay = 10 // Seconds const AllowableClockDrift = BlockDelay * 2 From bcf9778b18f014bbf04bcc5581ba0747191885da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 19:02:28 +0100 Subject: [PATCH 135/230] Update cbor-gen post-merge --- chain/actors/cbor_gen.go | 490 ++++++++++++++++++++++++++++++--------- chain/types/cbor_gen.go | 16 +- storage/cbor_gen.go | 63 ++--- 3 files changed, 426 insertions(+), 143 deletions(-) diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 304722e45..073286eb3 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -197,24 +197,28 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{140}); err != nil { + if _, err := w.Write([]byte{139}); err != nil { return err } - // t.t.Info (cid.Cid) (struct) - - if err := cbg.WriteCid(w, t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) - } - - // t.t.DePledgedCollateral (types.BigInt) (struct) - if err := t.DePledgedCollateral.MarshalCBOR(w); err != nil { + // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) (map) + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { return err } - // t.t.DePledgeTime (types.BigInt) (struct) - if err := t.DePledgeTime.MarshalCBOR(w); err != nil { - return err + for k, v := range t.PreCommittedSectors { + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + } // t.t.Sectors (cid.Cid) (struct) @@ -229,6 +233,12 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.ProvingSet: %w", err) } + // t.t.Info (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + // t.t.CurrentFaultSet (types.BitField) (struct) if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { return err @@ -277,39 +287,62 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 12 { + if extra != 11 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Info (cid.Cid) (struct) - - { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) - } - - t.Info = c + // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) (map) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } - // t.t.DePledgedCollateral (types.BigInt) (struct) - - { - - if err := t.DePledgedCollateral.UnmarshalCBOR(br); err != nil { - return err - } - + if maj != cbg.MajMap { + return fmt.Errorf("expected a map (major type 5)") + } + if extra > 4096 { + return fmt.Errorf("t.PreCommittedSectors: map too large") } - // t.t.DePledgeTime (types.BigInt) (struct) - { + t.PreCommittedSectors = make(map[string]*UnprovenSector, extra) - if err := t.DePledgeTime.UnmarshalCBOR(br); err != nil { - return err + for i, l := 0, int(extra); i < l; i++ { + + var k string + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + k = string(sval) } + var v *UnprovenSector + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + v = new(UnprovenSector) + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + + t.PreCommittedSectors[k] = v + } // t.t.Sectors (cid.Cid) (struct) @@ -334,6 +367,18 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.ProvingSet = c + } + // t.t.Info (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = c + } // t.t.CurrentFaultSet (types.BitField) (struct) @@ -492,12 +537,12 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { +func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } @@ -522,24 +567,6 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Proof ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { - return err - } - if _, err := w.Write(t.Proof); err != nil { - return err - } - - // t.t.DealIDs ([]uint64) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { - return err - } - for _, v := range t.DealIDs { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { - return err - } - } - // t.t.SectorNumber (uint64) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { return err @@ -547,7 +574,7 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { +func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -558,7 +585,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -606,53 +633,6 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.Epoch = uint64(extra) - // t.t.Proof ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Proof = make([]byte, extra) - if _, err := io.ReadFull(br, t.Proof); err != nil { - return err - } - // t.t.DealIDs ([]uint64) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.DealIDs: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.DealIDs = make([]uint64, extra) - } - for i := 0; i < int(extra); i++ { - - maj, val, err := cbg.CborReadHeader(br) - if err != nil { - return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) - } - - if maj != cbg.MajUnsignedInt { - return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) - } - - t.DealIDs[i] = val - } - // t.t.SectorNumber (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) @@ -666,6 +646,115 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *UnprovenSector) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{132}); err != nil { + return err + } + + // t.t.CommD ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { + return err + } + if _, err := w.Write(t.CommD); err != nil { + return err + } + + // t.t.CommR ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { + return err + } + if _, err := w.Write(t.CommR); err != nil { + return err + } + + // t.t.SubmitHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SubmitHeight))); err != nil { + return err + } + + // t.t.TicketEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TicketEpoch))); err != nil { + return err + } + return nil +} + +func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.CommD ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommD: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommD = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommD); err != nil { + return err + } + // t.t.CommR ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommR: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommR = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommR); err != nil { + return err + } + // t.t.SubmitHeight (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SubmitHeight = uint64(extra) + // t.t.TicketEpoch (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.TicketEpoch = uint64(extra) + return nil +} + func (t *MinerInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -3598,3 +3687,196 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { t.ActivationEpoch = uint64(extra) return nil } + +func (t *ComputeDataCommitmentParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + + // t.t.SectorSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil { + return err + } + return nil +} + +func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + // t.t.SectorSize (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorSize = uint64(extra) + return nil +} + +func (t *SectorProveCommitInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Proof ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + return nil +} + +func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Proof ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.t.SectorID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + return nil +} diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 2ac856520..fa3cddaf4 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -1480,7 +1480,7 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { return err } - // t.t.Cids ([]cid.Cid) + // t.t.Cids ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Cids)))); err != nil { return err } @@ -1490,7 +1490,7 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.Blocks ([]*types.BlockHeader) + // t.t.Blocks ([]*types.BlockHeader) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Blocks)))); err != nil { return err } @@ -1500,8 +1500,8 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.Height (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Height)); err != nil { + // t.t.Height (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Height))); err != nil { return err } return nil @@ -1522,7 +1522,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Cids ([]cid.Cid) + // t.t.Cids ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1547,7 +1547,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { t.Cids[i] = c } - // t.t.Blocks ([]*types.BlockHeader) + // t.t.Blocks ([]*types.BlockHeader) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1573,7 +1573,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { t.Blocks[i] = &v } - // t.t.Height (uint64) + // t.t.Height (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -1582,6 +1582,6 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Height = extra + t.Height = uint64(extra) return nil } diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 38e70c020..74c5f93e6 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -2,10 +2,11 @@ package storage import ( "fmt" + "io" + "github.com/filecoin-project/lotus/chain/types" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - "io" ) /* This file was generated by github.com/whyrusleeping/cbor-gen */ @@ -21,12 +22,12 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { return err } - // t.t.BlockHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.BlockHeight)); err != nil { + // t.t.BlockHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil { return err } - // t.t.TicketBytes ([]uint8) + // t.t.TicketBytes ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { return err } @@ -51,7 +52,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.BlockHeight (uint64) + // t.t.BlockHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -60,8 +61,8 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.BlockHeight = extra - // t.t.TicketBytes ([]uint8) + t.BlockHeight = uint64(extra) + // t.t.TicketBytes ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -90,17 +91,17 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.State (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.State)); err != nil { + // t.t.State (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil { return err } - // t.t.SectorID (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.SectorID)); err != nil { + // t.t.SectorID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { return err } - // t.t.CommD ([]uint8) + // t.t.CommD ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { return err } @@ -108,7 +109,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.CommR ([]uint8) + // t.t.CommR ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { return err } @@ -116,12 +117,12 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Ticket (storage.SealTicket) + // t.t.Ticket (storage.SealTicket) (struct) if err := t.Ticket.MarshalCBOR(w); err != nil { return err } - // t.t.PreCommitMessage (cid.Cid) + // t.t.PreCommitMessage (cid.Cid) (struct) if t.PreCommitMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -133,17 +134,17 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.RandHeight (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.RandHeight)); err != nil { + // t.t.RandHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RandHeight))); err != nil { return err } - // t.t.RandTs (types.TipSet) + // t.t.RandTs (types.TipSet) (struct) if err := t.RandTs.MarshalCBOR(w); err != nil { return err } - // t.t.CommitMessage (cid.Cid) + // t.t.CommitMessage (cid.Cid) (struct) if t.CommitMessage == nil { if _, err := w.Write(cbg.CborNull); err != nil { @@ -173,7 +174,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.State (uint64) + // t.t.State (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -182,8 +183,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.State = extra - // t.t.SectorID (uint64) + t.State = uint64(extra) + // t.t.SectorID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -192,8 +193,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SectorID = extra - // t.t.CommD ([]uint8) + t.SectorID = uint64(extra) + // t.t.CommD ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -210,7 +211,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommD); err != nil { return err } - // t.t.CommR ([]uint8) + // t.t.CommR ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -227,7 +228,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } - // t.t.Ticket (storage.SealTicket) + // t.t.Ticket (storage.SealTicket) (struct) { @@ -236,7 +237,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.PreCommitMessage (cid.Cid) + // t.t.PreCommitMessage (cid.Cid) (struct) { @@ -260,7 +261,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.RandHeight (uint64) + // t.t.RandHeight (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -269,8 +270,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.RandHeight = extra - // t.t.RandTs (types.TipSet) + t.RandHeight = uint64(extra) + // t.t.RandTs (types.TipSet) (struct) { @@ -291,7 +292,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.CommitMessage (cid.Cid) + // t.t.CommitMessage (cid.Cid) (struct) { From 3020f7a203973bc0a33857016d953b7480c8f3f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 19:19:57 +0100 Subject: [PATCH 136/230] Fix statestore.List --- lib/statestore/store.go | 2 +- lib/statestore/store_test.go | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 lib/statestore/store_test.go diff --git a/lib/statestore/store.go b/lib/statestore/store.go index f578c17e3..ca924d66e 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -136,7 +136,7 @@ func (st *StateStore) List(out interface{}) error { return err } - rout.Set(reflect.Append(rout.Elem(), elem.Elem())) + rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem())) } return nil diff --git a/lib/statestore/store_test.go b/lib/statestore/store_test.go new file mode 100644 index 000000000..667805b0c --- /dev/null +++ b/lib/statestore/store_test.go @@ -0,0 +1,38 @@ +package statestore + +import ( + "testing" + + "github.com/ipfs/go-datastore" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborrpc" +) + +func TestList(t *testing.T) { + ds := datastore.NewMapDatastore() + + e, err := cborrpc.Dump(types.NewInt(7)) + if err != nil { + t.Fatal(err) + } + + if err := ds.Put(datastore.NewKey("/2"), e); err != nil { + t.Fatal(err) + } + + st := &StateStore{ds:ds} + + var out []types.BigInt + if err := st.List(&out); err != nil { + t.Fatal(err) + } + + if len(out) != 1 { + t.Fatal("wrong len") + } + + if out[0].Int64() != 7 { + t.Fatal("wrong data") + } +} From 630134486da0308cfba429c7705e2734a819ba05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 5 Nov 2019 19:40:51 +0100 Subject: [PATCH 137/230] Mostly fix deals --- chain/deals/provider.go | 7 ++-- chain/deals/provider_states.go | 57 ++++++++---------------------- lib/jsonrpc/client.go | 4 +-- lib/sectorbuilder/mock.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 4 +-- lib/statestore/store_test.go | 2 +- 6 files changed, 25 insertions(+), 51 deletions(-) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 02966c05d..18420fb14 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -2,7 +2,6 @@ package deals import ( "context" - "github.com/filecoin-project/lotus/lib/statestore" "sync" cid "github.com/ipfs/go-cid" @@ -17,8 +16,10 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sectorblocks" ) type MinerDeal struct { @@ -42,6 +43,7 @@ type Provider struct { ask *types.SignedStorageAsk askLk sync.Mutex + secb *sectorblocks.SectorBlocks sminer *storage.Miner full api.FullNode @@ -69,7 +71,7 @@ type minerDealUpdate struct { mut func(*MinerDeal) } -func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { +func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) { addr, err := ds.Get(datastore.NewKey("miner-address")) if err != nil { return nil, err @@ -83,6 +85,7 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, dag dtypes.Staging sminer: sminer, dag: dag, full: fullNode, + secb: secb, pricePerByteBlock: types.NewInt(3), // TODO: allow setting minPieceSize: 1, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index c43628d13..9f5964cab 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -228,54 +227,26 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) if err != nil { return nil, err } - _ = pcid - /* - sectorID, err := p.sminer.AddUnixfsPiece(pcid, uf, deal.DealID) - if err != nil { - return nil, xerrors.Errorf("AddPiece failed: %s", err) - } - log.Warnf("New Sector: %d", sectorID) + sectorID, err := p.secb.AddUnixfsPiece(pcid, uf, deal.DealID) + if err != nil { + return nil, xerrors.Errorf("AddPiece failed: %s", err) + } + log.Warnf("New Sector: %d", sectorID) + + return func(deal *MinerDeal) { + deal.SectorID = sectorID + }, nil - return func(deal *MinerDeal) { - deal.SectorID = sectorID - }, nil - */ - panic("fixme") } // SEALING -func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) { - panic("fixme") - - /* - status, err := p.sminer.WaitSeal(ctx, deal.SectorID) - if err != nil { - return sectorbuilder.SectorSealingStatus{}, err - } - - switch status.State { - case sealing_state.Sealed: - case sealing_state.Failed: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg) - case sealing_state.Pending: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID) - case sealing_state.Sealing: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID) - default: - return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID) - } - - return status, nil - */ - -} - func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { err := p.sendSignedResponse(&Response{ State: api.DealSealing, Proposal: deal.ProposalCid, + // TODO: Send sector ID }) if err != nil { log.Warnf("Sending deal response failed: %s", err) @@ -284,11 +255,12 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { return nil, xerrors.Errorf("sealing sector failed: %w", err) } + // TODO: Let's not care after this point, for now at least, client can watch the chain - _, err = p.waitSealed(ctx, deal) + /*_, err = p.waitSealed(ctx, deal) if err != nil { return nil, err - } + }*/ // TODO: Spec doesn't say anything about inclusion proofs anywhere // Not sure what mechanisms prevents miner from storing data that isn't // clients' data @@ -297,13 +269,12 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal } func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - // TODO: Add dealID to commtracker (probably before sealing) /*mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) if err != nil { log.Warnf("Waiting for sector commitment message: %s", err) }*/ - panic("fixme") + //panic("fixme") /*err = p.sendSignedResponse(&Response{ State: api.DealComplete, diff --git a/lib/jsonrpc/client.go b/lib/jsonrpc/client.go index 81cf4ec0a..45b52c0e0 100644 --- a/lib/jsonrpc/client.go +++ b/lib/jsonrpc/client.go @@ -177,9 +177,9 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int) front := buf.Front() bufLk.Unlock() - cases := []reflect.SelectCase{ + cases := []reflect.SelectCase{ { - Dir: reflect.SelectRecv, + Dir: reflect.SelectRecv, Chan: reflect.ValueOf(chCtx.Done()), }, { diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 619ccb988..451757b17 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -32,7 +32,7 @@ func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { CacheDir: cache, WorkerThreads: 2, - Miner: addr, + Miner: addr, }) if err != nil { return nil, nil, err diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index e773988e0..9484823b0 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -71,8 +71,8 @@ func New(cfg *Config) (*SectorBuilder, error) { } return &SectorBuilder{ - handle: sbp, - Miner: cfg.Miner, + handle: sbp, + Miner: cfg.Miner, rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), }, nil } diff --git a/lib/statestore/store_test.go b/lib/statestore/store_test.go index 667805b0c..0d17ef20d 100644 --- a/lib/statestore/store_test.go +++ b/lib/statestore/store_test.go @@ -21,7 +21,7 @@ func TestList(t *testing.T) { t.Fatal(err) } - st := &StateStore{ds:ds} + st := &StateStore{ds: ds} var out []types.BigInt if err := st.List(&out); err != nil { From b447b6cf0ab35b80fc94f5000c2299ee5680cb50 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 10:50:27 -0800 Subject: [PATCH 138/230] only trim zeros to the right of the decimal --- chain/types/bigint_test.go | 17 +++++++++++++++++ chain/types/fil.go | 5 ++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/chain/types/bigint_test.go b/chain/types/bigint_test.go index 7e2cd71f4..5209e7051 100644 --- a/chain/types/bigint_test.go +++ b/chain/types/bigint_test.go @@ -32,3 +32,20 @@ func TestBigIntSerializationRoundTrip(t *testing.T) { } } + +func TestFilRoundTrip(t *testing.T) { + testValues := []string{ + "0", "1", "1.001", "100.10001", "101100", "5000.01", "5000", + } + + for _, v := range testValues { + fval, err := ParseFIL(v) + if err != nil { + t.Fatal(err) + } + + if fval.String() != v { + t.Fatal("mismatch in values!", v, fval.String()) + } + } +} diff --git a/chain/types/fil.go b/chain/types/fil.go index f44336fb5..80de6ced3 100644 --- a/chain/types/fil.go +++ b/chain/types/fil.go @@ -12,7 +12,10 @@ type FIL BigInt func (f FIL) String() string { r := new(big.Rat).SetFrac(f.Int, big.NewInt(build.FilecoinPrecision)) - return strings.TrimRight(r.FloatString(18), "0.") + if r.Sign() == 0 { + return "0" + } + return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".") } func (f FIL) Format(s fmt.State, ch rune) { From ba1e8c66684773ec8a6942d0edfa9bd434d7fc38 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 13:10:20 -0800 Subject: [PATCH 139/230] add proper piece padding in AddPiece --- storage/sector/store.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/storage/sector/store.go b/storage/sector/store.go index 52c45deef..40192b49d 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "math" "sync" "github.com/filecoin-project/go-sectorbuilder/sealing_state" @@ -59,8 +60,33 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er return &status, nil } +func computePaddedSize(size uint64) uint64 { + // TODO: there is a better way to compute the log2 of an integer... i'm just lazy + logv := uint64(math.Log2(float64(size))) + + sectSize := uint64(1 << (logv + 1)) + bound := sectorbuilder.UserBytesForSectorSize(sectSize) + if size <= bound { + return bound + } + + return sectorbuilder.UserBytesForSectorSize(1 << (logv + 2)) +} + +type nullReader struct{} + +func (nr nullReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - sectorID, err = s.sb.AddPiece(ref, size, r) + padSize := computePaddedSize(size) + + r = io.MultiReader(r, io.LimitReader(nullReader{}, int64(padSize-size))) + sectorID, err = s.sb.AddPiece(ref, padSize, r) if err != nil { return 0, err } From 87616e1c0322316a183d1362cad7f1053dbfe2f9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 15:57:46 -0800 Subject: [PATCH 140/230] use better math --- storage/sector/store.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/sector/store.go b/storage/sector/store.go index 40192b49d..0112e3f4a 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -1,10 +1,11 @@ package sector import ( + "bytes" "context" "fmt" "io" - "math" + "math/bits" "sync" "github.com/filecoin-project/go-sectorbuilder/sealing_state" @@ -61,16 +62,15 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er } func computePaddedSize(size uint64) uint64 { - // TODO: there is a better way to compute the log2 of an integer... i'm just lazy - logv := uint64(math.Log2(float64(size))) + logv := 64 - bits.LeadingZeros64(size) - sectSize := uint64(1 << (logv + 1)) + sectSize := uint64(1 << logv) bound := sectorbuilder.UserBytesForSectorSize(sectSize) if size <= bound { return bound } - return sectorbuilder.UserBytesForSectorSize(1 << (logv + 2)) + return sectorbuilder.UserBytesForSectorSize(1 << (logv + 1)) } type nullReader struct{} From 1fc0971db07c17398389a2cd3496da6cdf242bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 13:03:36 +0100 Subject: [PATCH 141/230] sectorstore: test computePaddedSize --- storage/sector/store.go | 1 - storage/sector/store_test.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 storage/sector/store_test.go diff --git a/storage/sector/store.go b/storage/sector/store.go index 0112e3f4a..64260b9f9 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -1,7 +1,6 @@ package sector import ( - "bytes" "context" "fmt" "io" diff --git a/storage/sector/store_test.go b/storage/sector/store_test.go new file mode 100644 index 000000000..5d473ba18 --- /dev/null +++ b/storage/sector/store_test.go @@ -0,0 +1,12 @@ +package sector + +import ( + "gotest.tools/assert" + "testing" +) + +func TestComputePaddedSize(t *testing.T) { + assert.Equal(t, uint64(1040384), computePaddedSize(1000000)) + assert.Equal(t, uint64(1016), computePaddedSize(548)) + assert.Equal(t, uint64(4064), computePaddedSize(2048)) +} \ No newline at end of file From be58a51f9f673a8e6291d18cc4600d72b0fe57dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 13:03:59 +0100 Subject: [PATCH 142/230] paramfetch: env var to not check param files each time --- build/paramfetch.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/paramfetch.go b/build/paramfetch.go index 39e6ffda6..5b1f4068d 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -91,6 +91,10 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { } func (ft *fetch) checkFile(path string, info paramFile) error { + if os.Getenv("TRUST_PARAMS") == "1" { + log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION") + } + f, err := os.Open(path) if err != nil { return err From cb3965bcf506c27311efa99bf3fdcaa117e09d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 13:04:33 +0100 Subject: [PATCH 143/230] Improve errors around deal handling --- chain/deals/client_utils.go | 2 +- chain/deals/provider.go | 2 +- chain/deals/provider_utils.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 11 ++++++++++- storage/sealing.go | 2 +- storage/sector/store_test.go | 4 ++-- storage/sectorblocks/blocks.go | 13 +++++++------ 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 9c26f4268..8a68f1cdd 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -25,7 +25,7 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { } // TODO: store in some sort of audit log - log.Errorf("deal %s failed: %s", id, cerr) + log.Errorf("deal %s failed: %+v", id, cerr) } func (c *Client) dataSize(ctx context.Context, data cid.Cid) (int64, error) { diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 18420fb14..608432352 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -162,7 +162,7 @@ func (p *Provider) onIncoming(deal MinerDeal) { func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { log.Infof("Deal %s updated state to %d", update.id, update.newState) if update.err != nil { - log.Errorf("deal %s (newSt: %d) failed: %s", update.id, update.newState, update.err) + log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) p.failDeal(update.id, update.err) return } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 5931cefc9..547ff8c76 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -26,7 +26,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l) } - log.Errorf("deal %s failed: %s", id, cerr) + log.Warnf("deal %s failed: %s", id, cerr) err := p.sendSignedResponse(&Response{ State: api.DealFailed, diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 9484823b0..58f37cf56 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -231,11 +231,20 @@ func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { go func() { defer wait.Unlock() - _, werr = io.CopyN(w, r, n) + copied, werr := io.CopyN(w, r, n) + if werr != nil { + log.Warnf("toReadableFile: copy error: %+v", werr) + } err := w.Close() if werr == nil { werr = err + log.Warnf("toReadableFile: close error: %+v", err) + return + } + if copied != n { + log.Warnf("copied different amount than expected: %d != %d", copied, n) + werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) } }() diff --git a/storage/sealing.go b/storage/sealing.go index d1883be3d..625a35c1b 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -114,7 +114,7 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { } func (m *Miner) failSector(id uint64, err error) { - log.Error(err) + log.Errorf("sector %d error: %+v", id, err) panic(err) // todo: better error handling strategy } diff --git a/storage/sector/store_test.go b/storage/sector/store_test.go index 5d473ba18..a2e1d00ac 100644 --- a/storage/sector/store_test.go +++ b/storage/sector/store_test.go @@ -5,8 +5,8 @@ import ( "testing" ) -func TestComputePaddedSize(t *testing.T) { +func TestComputePaddedSize(t *testing.T) { assert.Equal(t, uint64(1040384), computePaddedSize(1000000)) assert.Equal(t, uint64(1016), computePaddedSize(548)) assert.Equal(t, uint64(4064), computePaddedSize(2048)) -} \ No newline at end of file +} diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 28b864e7a..4f1049f23 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -11,6 +11,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfs" + "golang.org/x/xerrors" "sync" "github.com/ipfs/go-cid" @@ -89,13 +90,13 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si err = nil } if err != nil { - return err + return xerrors.Errorf("getting existing refs: %w", err) } var refs []api.SealedRef if len(v) > 0 { if err := cbor.DecodeInto(v, &refs); err != nil { - return err + return xerrors.Errorf("decoding existing refs: %w", err) } } @@ -107,7 +108,7 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si newRef, err := cbor.DumpObject(&refs) if err != nil { - return err + return xerrors.Errorf("serializing refs: %w", err) } return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow } @@ -128,20 +129,20 @@ func (r *refStorer) Read(p []byte) (n int, err error) { for { data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { - return 0, err + return 0, xerrors.Errorf("reading block: %w", err) } if len(data) == 0 { // TODO: batch // TODO: GC if err := r.intermediate.Put(nd); err != nil { - return 0, err + return 0, xerrors.Errorf("storing intermediate node: %w", err) } continue } if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil { - return 0, err + return 0, xerrors.Errorf("writing ref: %w", err) } read := copy(p, data) From 40b1f918433fcf6efd5bc801cca0118112124952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 13:22:08 +0100 Subject: [PATCH 144/230] fix sector block ref serialization --- api/api_storage.go | 6 +- api/cbor_gen.go | 142 +++++++++++++++++++++++++++++++++ gen/main.go | 2 + storage/sectorblocks/blocks.go | 47 +++++------ 4 files changed, 173 insertions(+), 24 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 7a392455f..75fc55849 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -69,5 +69,9 @@ type StorageMiner interface { type SealedRef struct { Piece string Offset uint64 - Size uint32 + Size uint64 +} + +type SealedRefs struct { + Refs []SealedRef } diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 6f71bbaed..0f968865b 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -127,3 +127,145 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { return nil } + +func (t *SealedRef) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.Piece (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Piece)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Piece)); err != nil { + return err + } + + // t.t.Offset (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + return nil +} + +func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Piece (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Piece = string(sval) + } + // t.t.Offset (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + return nil +} + +func (t *SealedRefs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Refs ([]api.SealedRef) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil { + return err + } + for _, v := range t.Refs { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Refs ([]api.SealedRef) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Refs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Refs = make([]SealedRef, extra) + } + for i := 0; i < int(extra); i++ { + + var v SealedRef + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Refs[i] = v + } + + return nil +} diff --git a/gen/main.go b/gen/main.go index 1aac6ef06..296b7bbf4 100644 --- a/gen/main.go +++ b/gen/main.go @@ -48,6 +48,8 @@ func main() { err = gen.WriteTupleEncodersToFile("./api/cbor_gen.go", "api", api.PaymentInfo{}, + api.SealedRef{}, + api.SealedRefs{}, ) if err != nil { fmt.Println(err) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 4f1049f23..d8666b41c 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -1,25 +1,26 @@ package sectorblocks import ( + "bytes" "context" "errors" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-datastore/query" - blockstore "github.com/ipfs/go-ipfs-blockstore" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfs" - "golang.org/x/xerrors" "sync" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" files "github.com/ipfs/go-ipfs-files" - cbor "github.com/ipfs/go-ipld-cbor" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-unixfs" + "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sector" ) @@ -74,14 +75,14 @@ type UnixfsReader interface { type refStorer struct { blockReader UnixfsReader - writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint32) error + writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint64) error intermediate blockstore.Blockstore pieceRef string remaining []byte } -func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint32) error { +func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint64) error { st.keyLk.Lock() // TODO: make this multithreaded defer st.keyLk.Unlock() @@ -93,20 +94,20 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si return xerrors.Errorf("getting existing refs: %w", err) } - var refs []api.SealedRef + var refs api.SealedRefs if len(v) > 0 { - if err := cbor.DecodeInto(v, &refs); err != nil { + if err := cborrpc.ReadCborRPC(bytes.NewReader(v), &refs); err != nil { return xerrors.Errorf("decoding existing refs: %w", err) } } - refs = append(refs, api.SealedRef{ + refs.Refs = append(refs.Refs, api.SealedRef{ Piece: pieceRef, Offset: offset, Size: size, }) - newRef, err := cbor.DumpObject(&refs) + newRef, err := cborrpc.Dump(&refs) if err != nil { return xerrors.Errorf("serializing refs: %w", err) } @@ -141,7 +142,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { continue } - if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil { + if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint64(len(data))); err != nil { return 0, xerrors.Errorf("writing ref: %w", err) } @@ -188,12 +189,12 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { return nil, err } - var refs []api.SealedRef - if err := cbor.DecodeInto(ent.Value, &refs); err != nil { + var refs api.SealedRefs + if err := cborrpc.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil { return nil, err } - out[refCid] = refs + out[refCid] = refs.Refs } return out, nil @@ -208,12 +209,12 @@ func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: t return nil, err } - var refs []api.SealedRef - if err := cbor.DecodeInto(ent, &refs); err != nil { + var refs api.SealedRefs + if err := cborrpc.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil { return nil, err } - return refs, nil + return refs.Refs, nil } func (st *SectorBlocks) GetSize(k cid.Cid) (uint64, error) { From 58229defcd5af18c8587250010860e7cefe5a750 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 6 Nov 2019 16:10:27 +0100 Subject: [PATCH 145/230] Fix faucet ip limiting to use X-Real-IP License: MIT Signed-off-by: Jakub Sztandera --- cmd/lotus-fountain/main.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-fountain/main.go b/cmd/lotus-fountain/main.go index 6d590bb0c..bd8127683 100644 --- a/cmd/lotus-fountain/main.go +++ b/cmd/lotus-fountain/main.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "net" "net/http" "os" "strconv" @@ -24,7 +25,7 @@ import ( var log = logging.Logger("main") -var sendPerRequest = types.NewInt(500_000_000) +var sendPerRequest, _ = types.ParseFIL("0.005") func main() { logging.SetLogLevel("*", "INFO") @@ -104,7 +105,7 @@ var runCmd = &cli.Command{ TotalRate: time.Second, TotalBurst: 20, IPRate: 10 * time.Minute, - IPBurst: 1, + IPBurst: 2, WalletRate: 1 * time.Hour, WalletBurst: 1, }), @@ -151,7 +152,20 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { } // Limit based on IP - limiter = h.limiter.GetIPLimiter(r.RemoteAddr) + + reqIP := r.Header.Get("X-Real-IP") + if reqIP == "" { + h, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + log.Errorf("could not get ip from: %s, err: %s", r.RemoteAddr, err) + } + reqIP = h + } + if i := net.ParseIP(reqIP); i != nil && i.IsLoopback() { + log.Errorf("rate limiting localhost: %s", reqIP) + } + + limiter = h.limiter.GetIPLimiter(reqIP) if !limiter.Allow() { http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) return @@ -164,7 +178,7 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) { } smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ - Value: sendPerRequest, + Value: types.BigInt(sendPerRequest), From: h.from, To: to, @@ -229,7 +243,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) { } smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{ - Value: sendPerRequest, + Value: types.BigInt(sendPerRequest), From: h.from, To: owner, From 2b57fd14f10aa9004b7bedbd3fae811ec63d21e7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 6 Nov 2019 16:11:19 +0100 Subject: [PATCH 146/230] Fix temporal problems in bad block cache License: MIT Signed-off-by: Jakub Sztandera --- chain/sync.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chain/sync.go b/chain/sync.go index dac2a79e5..8c438b0c0 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -2,6 +2,7 @@ package chain import ( "context" + "errors" "fmt" "sync" "time" @@ -379,7 +380,9 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) for _, b := range fts.Blocks { if err := syncer.ValidateBlock(ctx, b); err != nil { - syncer.bad.Add(b.Cid()) + if !errors.Is(err, ErrTemporal) { + syncer.bad.Add(b.Cid()) + } return xerrors.Errorf("validating block %s: %w", b.Cid(), err) } @@ -444,6 +447,8 @@ func (syncer *Syncer) validateTickets(ctx context.Context, mworker address.Addre return nil } +var ErrTemporal = errors.New("temporal error") + // Should match up with 'Semantical Validation' in validation.md in the spec func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error { ctx, span := trace.StartSpan(ctx, "validateBlock") @@ -470,7 +475,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } if h.Timestamp > uint64(time.Now().Unix()+build.AllowableClockDrift) { - return xerrors.Errorf("block was from the future") + return xerrors.Errorf("block was from the future: %w", ErrTemporal) } if h.Timestamp < baseTs.MinTimestamp()+uint64(build.BlockDelay*len(h.Tickets)) { From 68c2d4f58a1369a1ba72f42b8291783c228276f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 18:38:42 +0100 Subject: [PATCH 147/230] deals: Set correct Refs --- chain/deals/cbor_gen.go | 37 ++++++++++++++++++++++++++++++--- chain/deals/client.go | 23 ++++++++++++-------- chain/deals/client_utils.go | 24 ++++++++++++++++----- chain/deals/provider.go | 11 +++------- chain/deals/provider_states.go | 8 +------ chain/deals/provider_utils.go | 9 ++++---- chain/deals/types.go | 8 ++++--- lib/padreader/padreader.go | 38 ++++++++++++++++++++++++++++++++++ node/impl/client/client.go | 1 - storage/sector/store.go | 27 +----------------------- storage/sectorblocks/blocks.go | 4 ++++ 11 files changed, 123 insertions(+), 67 deletions(-) create mode 100644 lib/padreader/padreader.go diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index ec46764f9..08edb9371 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -118,7 +118,7 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{129}); err != nil { + if _, err := w.Write([]byte{130}); err != nil { return err } @@ -126,6 +126,13 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { if err := t.DealProposal.MarshalCBOR(w); err != nil { return err } + + // t.t.Piece (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Piece); err != nil { + return xerrors.Errorf("failed to write cid field t.Piece: %w", err) + } + return nil } @@ -140,7 +147,7 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 1 { + if extra != 2 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -148,9 +155,33 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { { - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + pb, err := br.PeekByte() + if err != nil { return err } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.DealProposal = new(actors.StorageDealProposal) + if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + return err + } + } + + } + // t.t.Piece (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Piece: %w", err) + } + + t.Piece = c } return nil diff --git a/chain/deals/client.go b/chain/deals/client.go index e89a9886f..b99422e11 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -201,25 +201,25 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro } } - dataSize, err := c.dataSize(ctx, p.Data) + commP, pieceSize, err := c.commP(ctx, p.Data) - proposal := &actors.StorageDealProposal{ - PieceRef: p.Data.Bytes(), - PieceSize: uint64(dataSize), + dealProposal := &actors.StorageDealProposal{ + PieceRef: commP, + PieceSize: uint64(pieceSize), PieceSerialization: actors.SerializationUnixFSv0, Client: p.Client, Provider: p.ProviderAddress, ProposalExpiration: p.ProposalExpiration, Duration: p.Duration, StoragePricePerEpoch: p.PricePerEpoch, - StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc + StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc } - if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil { + if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil { return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } - proposalNd, err := cborrpc.AsIpld(proposal) + proposalNd, err := cborrpc.AsIpld(dealProposal) if err != nil { return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } @@ -230,6 +230,11 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } + proposal := &Proposal{ + DealProposal: dealProposal, + Piece: p.Data, + } + if err := cborrpc.WriteCborRPC(s, proposal); err != nil { s.Reset() return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) @@ -237,7 +242,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro deal := &ClientDeal{ ProposalCid: proposalNd.Cid(), - Proposal: *proposal, + Proposal: *dealProposal, State: api.DealUnknown, Miner: p.MinerID, @@ -247,7 +252,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro c.incoming <- deal return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{ - Address: proposal.Provider, + Address: dealProposal.Provider, ID: deal.Miner, }) } diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 8a68f1cdd..135f696a7 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -10,6 +10,8 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/padreader" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (c *Client) failDeal(id cid.Cid, cerr error) { @@ -28,26 +30,38 @@ func (c *Client) failDeal(id cid.Cid, cerr error) { log.Errorf("deal %s failed: %+v", id, cerr) } -func (c *Client) dataSize(ctx context.Context, data cid.Cid) (int64, error) { +func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) { root, err := c.dag.Get(ctx, data) if err != nil { log.Errorf("failed to get file root for deal: %s", err) - return 0, err + return nil, 0, err } n, err := unixfile.NewUnixfsFile(ctx, c.dag, root) if err != nil { log.Errorf("cannot open unixfs file: %s", err) - return 0, err + return nil, 0, err } uf, ok := n.(files.File) if !ok { // TODO: we probably got directory, how should we handle this in unixfs mode? - return 0, xerrors.New("unsupported unixfs type") + return nil, 0, xerrors.New("unsupported unixfs type") } - return uf.Size() + s, err := uf.Size() + if err != nil { + return nil, 0, err + } + + pr, psize := padreader.New(uf, uint64(s)) + + commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize) + if err != nil { + return nil, 0, xerrors.Errorf("generating CommP: %w", err) + } + + return commp[:], psize, nil } func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 608432352..ecabc7d13 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -192,24 +192,19 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } } -func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) { +func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { proposalNd, err := cborrpc.AsIpld(&proposal) if err != nil { return MinerDeal{}, err } - ref, err := cid.Cast(proposal.PieceRef) - if err != nil { - return MinerDeal{}, err - } - return MinerDeal{ Client: s.Conn().RemotePeer(), - Proposal: proposal, + Proposal: *proposal.DealProposal, ProposalCid: proposalNd.Cid(), State: api.DealUnknown, - Ref: ref, + Ref: proposal.Piece, s: s, }, nil diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 9f5964cab..ea5c6c772 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -4,7 +4,6 @@ import ( "bytes" "context" - "github.com/ipfs/go-cid" "github.com/ipfs/go-merkledag" unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" @@ -223,12 +222,7 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match unixfs file size") } - pcid, err := cid.Cast(deal.Proposal.PieceRef) - if err != nil { - return nil, err - } - - sectorID, err := p.secb.AddUnixfsPiece(pcid, uf, deal.DealID) + sectorID, err := p.secb.AddUnixfsPiece(deal.Ref, uf, deal.DealID) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 547ff8c76..2ea5dc05e 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -5,7 +5,6 @@ import ( "runtime" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -45,21 +44,21 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { } } -func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealProposal, err error) { +func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { if err := cborrpc.ReadCborRPC(s, &proposal); err != nil { log.Errorw("failed to read proposal message", "error", err) return proposal, err } - if err := proposal.Verify(); err != nil { + if err := proposal.DealProposal.Verify(); err != nil { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } // TODO: Validate proposal maybe // (and signature, obviously) - if proposal.Provider != p.actor { - log.Errorf("proposal with wrong ProviderAddress: %s", proposal.Provider) + if proposal.DealProposal.Provider != p.actor { + log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) return proposal, err } diff --git a/chain/deals/types.go b/chain/deals/types.go index 27bf31a8b..2177517ab 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -8,11 +8,13 @@ import ( "github.com/ipfs/go-cid" ) -const DealProtocolID = "/fil/storage/mk/1.0.0" -const AskProtocolID = "/fil/storage/ask/1.0.0" +const DealProtocolID = "/fil/storage/mk/1.0.1" +const AskProtocolID = "/fil/storage/ask/1.0.1" type Proposal struct { - DealProposal actors.StorageDealProposal + DealProposal *actors.StorageDealProposal + + Piece cid.Cid // Used for retrieving from the client } type Response struct { diff --git a/lib/padreader/padreader.go b/lib/padreader/padreader.go new file mode 100644 index 000000000..135e35b57 --- /dev/null +++ b/lib/padreader/padreader.go @@ -0,0 +1,38 @@ +package padreader + +import ( + "io" + "math/bits" + + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" +) + +func PaddedSize(size uint64) uint64 { + logv := 64 - bits.LeadingZeros64(size) + + sectSize := uint64(1 << logv) + bound := sectorbuilder.GetMaxUserBytesPerStagedSector(sectSize) + if size <= bound { + return bound + } + + return sectorbuilder.GetMaxUserBytesPerStagedSector(1 << (logv + 1)) +} + +type nullReader struct{} + +func (nr nullReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + +func New(r io.Reader, size uint64) (io.Reader, uint64) { + padSize := PaddedSize(size) + + return io.MultiReader( + io.LimitReader(r, int64(size)), + io.LimitReader(nullReader{}, int64(padSize-size)), + ), padSize +} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 85e9ca4f7..17ab26731 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -88,7 +88,6 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A } c, err := a.DealClient.Start(ctx, proposal) - // TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!) return &c, err } diff --git a/storage/sector/store.go b/storage/sector/store.go index 64260b9f9..52c45deef 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math/bits" "sync" "github.com/filecoin-project/go-sectorbuilder/sealing_state" @@ -60,32 +59,8 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er return &status, nil } -func computePaddedSize(size uint64) uint64 { - logv := 64 - bits.LeadingZeros64(size) - - sectSize := uint64(1 << logv) - bound := sectorbuilder.UserBytesForSectorSize(sectSize) - if size <= bound { - return bound - } - - return sectorbuilder.UserBytesForSectorSize(1 << (logv + 1)) -} - -type nullReader struct{} - -func (nr nullReader) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - padSize := computePaddedSize(size) - - r = io.MultiReader(r, io.LimitReader(nullReader{}, int64(padSize-size))) - sectorID, err = s.sb.AddPiece(ref, padSize, r) + sectorID, err = s.sb.AddPiece(ref, size, r) if err != nil { return 0, err } diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index d8666b41c..d46ef9f1b 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "io" "sync" "github.com/ipfs/go-cid" @@ -130,6 +131,9 @@ func (r *refStorer) Read(p []byte) (n int, err error) { for { data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { + if err == io.EOF { + return 0, io.EOF + } return 0, xerrors.Errorf("reading block: %w", err) } From 37721b2a2207ccb6df6f36e25d02e1aa92e9f6c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 6 Nov 2019 20:00:04 +0100 Subject: [PATCH 148/230] Get some deals to work --- chain/deals/provider_states.go | 5 +++-- lotuspond/front/src/Client.js | 4 ++-- storage/sectorblocks/blocks.go | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index ea5c6c772..5e5dbc273 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -218,8 +219,8 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) if err != nil { return nil, xerrors.Errorf("getting unixfs file size: %w", err) } - if uint64(size) != deal.Proposal.PieceSize { - return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match unixfs file size") + if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize { + return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") } sectorID, err := p.secb.AddUnixfsPiece(deal.Ref, uf, deal.DealID) diff --git a/lotuspond/front/src/Client.js b/lotuspond/front/src/Client.js index 37606223b..70aabfd6f 100644 --- a/lotuspond/front/src/Client.js +++ b/lotuspond/front/src/Client.js @@ -21,7 +21,7 @@ class Client extends React.Component { this.state = { miners: ["t0101"], - ask: {Price: "500000000"}, + ask: {Price: "1000000000"}, // 2x min default ask to account for bin packing (could also do the math correctly below, but..) kbs: 1, blocks: 12, @@ -52,7 +52,7 @@ class Client extends React.Component { update = (name) => (e) => this.setState({ [name]: e.target.value }); makeDeal = async () => { - let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) + let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) * 2 let file = await this.props.pondClient.call('Pond.CreateRandomFile', [this.state.kbs * 1000]) // 1024 won't fit in 1k blocks :( let cid = await this.props.client.call('Filecoin.ClientImport', [file]) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index d46ef9f1b..1d1289def 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -20,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sector" @@ -172,7 +173,9 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6 intermediate: st.intermediate, } - return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID) + pr, psize := padreader.New(r, uint64(size)) + + return st.Store.AddPiece(refst.pieceRef, psize, pr, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { From b3fe304d823804056ecd0e1a2e5f468df3a940cc Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 6 Nov 2019 20:26:01 +0100 Subject: [PATCH 149/230] add is permanent License: MIT Signed-off-by: Jakub Sztandera --- chain/sync.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chain/sync.go b/chain/sync.go index 8c438b0c0..55ac65719 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -369,6 +369,10 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { return nil } +func isPermanent(err error) bool { + return !errors.Is(err, ErrTemporal) +} + func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error { ctx, span := trace.StartSpan(ctx, "validateTipSet") defer span.End() @@ -380,7 +384,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) for _, b := range fts.Blocks { if err := syncer.ValidateBlock(ctx, b); err != nil { - if !errors.Is(err, ErrTemporal) { + if isPermanent(err) { syncer.bad.Add(b.Cid()) } return xerrors.Errorf("validating block %s: %w", b.Cid(), err) From 00114c21e5186b56b2ec5f20ab4991ac7011f444 Mon Sep 17 00:00:00 2001 From: shannonwells Date: Wed, 6 Nov 2019 11:36:40 -0800 Subject: [PATCH 150/230] fix linker failure for OpenCL lib --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdf73642..c1106b210 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,8 @@ commands: prepare: steps: - checkout + - run: sudo apt-get update + - run: sudo apt-get install ocl-icd-opencl-dev - run: git submodule sync - run: git submodule update --init download-params: From f7651f180b617ade834d9d9b83fb2f969ce97faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 00:09:48 +0100 Subject: [PATCH 151/230] Create filler deals --- build/paramfetch.go | 1 + build/params.go | 2 +- chain/deals/client_utils.go | 2 +- chain/deals/provider.go | 2 +- gen/main.go | 9 ++ lib/padreader/padreader_test.go | 12 +++ lib/sectorbuilder/sectorbuilder.go | 11 ++- storage/garbage.go | 153 +++++++++++++---------------- storage/sealing.go | 4 +- storage/sector/cbor_gen.go | 119 ++++++++++++++++++++++ storage/sector/store.go | 64 +++++++++--- storage/sector/store_test.go | 44 ++++++++- storage/sector_states.go | 28 +++++- 13 files changed, 343 insertions(+), 108 deletions(-) create mode 100644 lib/padreader/padreader_test.go create mode 100644 storage/sector/cbor_gen.go diff --git a/build/paramfetch.go b/build/paramfetch.go index 5b1f4068d..55dfd62bc 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -93,6 +93,7 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) { func (ft *fetch) checkFile(path string, info paramFile) error { if os.Getenv("TRUST_PARAMS") == "1" { log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION") + return nil } f, err := os.Open(path) diff --git a/build/params.go b/build/params.go index 22e1ece3c..cd2fc738c 100644 --- a/build/params.go +++ b/build/params.go @@ -37,7 +37,7 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // Consensus / Network // Seconds -const BlockDelay = 10 +const BlockDelay = 2 // Seconds const AllowableClockDrift = BlockDelay * 2 diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 135f696a7..7cdd77192 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -80,7 +80,7 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { // TODO: verify signature if resp.Response.Proposal != deal.ProposalCid { - return nil, xerrors.New("miner responded to a wrong proposal") + return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) } return &resp.Response, nil diff --git a/chain/deals/provider.go b/chain/deals/provider.go index ecabc7d13..46726298c 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -88,7 +88,7 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks secb: secb, pricePerByteBlock: types.NewInt(3), // TODO: allow setting - minPieceSize: 1, + minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up)) conns: map[cid.Cid]inet.Stream{}, diff --git a/gen/main.go b/gen/main.go index 296b7bbf4..4c0c2167a 100644 --- a/gen/main.go +++ b/gen/main.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" + "github.com/filecoin-project/lotus/storage/sector" ) func main() { @@ -158,4 +159,12 @@ func main() { fmt.Println(err) os.Exit(1) } + + err = gen.WriteTupleEncodersToFile("./storage/sector/cbor_gen.go", "sector", + sector.DealMapping{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } } diff --git a/lib/padreader/padreader_test.go b/lib/padreader/padreader_test.go new file mode 100644 index 000000000..bf8464806 --- /dev/null +++ b/lib/padreader/padreader_test.go @@ -0,0 +1,12 @@ +package padreader + +import ( + "gotest.tools/assert" + "testing" +) + +func TestComputePaddedSize(t *testing.T) { + assert.Equal(t, uint64(1040384), PaddedSize(1000000)) + assert.Equal(t, uint64(1016), PaddedSize(548)) + assert.Equal(t, uint64(4064), PaddedSize(2048)) +} diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 58f37cf56..7f09fa81b 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -40,6 +40,7 @@ const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer + ssize uint64 Miner address.Address @@ -71,7 +72,9 @@ func New(cfg *Config) (*SectorBuilder, error) { } return &SectorBuilder{ - handle: sbp, + handle: sbp, + ssize: cfg.SectorSize, + Miner: cfg.Miner, rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), }, nil @@ -172,6 +175,10 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed return sectorbuilder.GeneratePoSt(sb.handle, sectorInfo, challengeSeed, faults) } +func (sb *SectorBuilder) SectorSize() uint64 { + return sb.ssize +} + var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { @@ -237,7 +244,7 @@ func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { } err := w.Close() - if werr == nil { + if werr == nil && err != nil { werr = err log.Warnf("toReadableFile: close error: %+v", err) return diff --git a/storage/garbage.go b/storage/garbage.go index 24af346ec..8e3cf1a10 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -16,46 +16,13 @@ import ( "github.com/filecoin-project/lotus/lib/sectorbuilder" ) -func (m *Miner) StoreGarbageData(_ context.Context) error { - ctx := context.TODO() - ssize, err := m.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) - } - go func() { - size := sectorbuilder.UserBytesForSectorSize(ssize) - - /*// Add market funds - smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: m.worker, - Value: types.NewInt(size), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.AddBalance, - }) - if err != nil { - log.Error(err) - return - } - - r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - log.Error(err) - return - } - - if r.Receipt.ExitCode != 0 { - log.Error(xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)) - return - }*/ - // Publish a deal - - // TODO: Maybe cache +// TODO: expected sector ID +func (m *Miner) storeGarbage(ctx context.Context, sizes ...uint64) ([]uint64, error) { + deals := make([]actors.StorageDeal, len(sizes)) + for i, size := range sizes { commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) if err != nil { - log.Error(err) - return + return nil, err } sdp := actors.StorageDealProposal{ @@ -72,66 +39,86 @@ func (m *Miner) StoreGarbageData(_ context.Context) error { } if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil { - log.Error(xerrors.Errorf("signing storage deal failed: ", err)) - return + return nil, xerrors.Errorf("signing storage deal failed: ", err) } storageDeal := actors.StorageDeal{ Proposal: sdp, } if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil { - log.Error(xerrors.Errorf("signing storage deal failed: ", err)) - return + return nil, xerrors.Errorf("signing storage deal failed: ", err) } - params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDeal{storageDeal}, - }) - if err != nil { - log.Error(xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)) - } + deals[i] = storageDeal + } - // TODO: We may want this to happen after fetching data - smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: m.worker, - Value: types.NewInt(0), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.PublishStorageDeals, - Params: params, - }) - if err != nil { - log.Error(err) - return - } - r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - log.Error(err) - return - } - if r.Receipt.ExitCode != 0 { - log.Error(xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)) - } - var resp actors.PublishStorageDealResponse - if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { - log.Error(err) - return - } - if len(resp.DealIDs) != 1 { - log.Error("got unexpected number of DealIDs from") - return - } + params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{ + Deals: deals, + }) + if aerr != nil { + return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr) + } + // TODO: We may want this to happen after fetching data + smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: m.worker, + Value: types.NewInt(0), + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.PublishStorageDeals, + Params: params, + }) + if err != nil { + return nil, err + } + r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) + if err != nil { + return nil, err + } + if r.Receipt.ExitCode != 0 { + log.Error(xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)) + } + var resp actors.PublishStorageDealResponse + if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { + return nil, err + } + if len(resp.DealIDs) != len(sizes) { + return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals") + } + + sectorIDs := make([]uint64, len(sizes)) + + for i, size := range sizes { name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorId, err := m.secst.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), resp.DealIDs[0]) + sectorID, err := m.secst.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), resp.DealIDs[i]) if err != nil { - log.Error(err) + return nil, err + } + + sectorIDs[i] = sectorID + } + + return sectorIDs, nil +} + +func (m *Miner) StoreGarbageData(_ context.Context) error { + ctx := context.TODO() + ssize, err := m.SectorSize(ctx) + if err != nil { + return xerrors.Errorf("failed to get miner sector size: %w", err) + } + go func() { + size := sectorbuilder.UserBytesForSectorSize(ssize) + + sids, err := m.storeGarbage(ctx, size) + if err != nil { + log.Errorf("%+v", err) return } - if err := m.SealSector(context.TODO(), sectorId); err != nil { - log.Error(err) + if err := m.SealSector(context.TODO(), sids[0]); err != nil { + log.Errorf("%+v", err) return } }() diff --git a/storage/sealing.go b/storage/sealing.go index 625a35c1b..637eb8da2 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -71,7 +71,7 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { go func() { select { case m.sectorUpdated <- sectorUpdate{ - newState: api.Unsealed, + newState: api.Packing, id: sector.SectorID, }: case <-m.stop: @@ -102,6 +102,8 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { } switch update.newState { + case api.Packing: + m.handle(ctx, sector, m.finishPacking, api.Unsealed) case api.Unsealed: m.handle(ctx, sector, m.sealPreCommit, api.PreCommitting) case api.PreCommitting: diff --git a/storage/sector/cbor_gen.go b/storage/sector/cbor_gen.go new file mode 100644 index 000000000..11a2365f1 --- /dev/null +++ b/storage/sector/cbor_gen.go @@ -0,0 +1,119 @@ +package sector + +import ( + "fmt" + "io" + + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +/* This file was generated by github.com/whyrusleeping/cbor-gen */ + +var _ = xerrors.Errorf + +func (t *DealMapping) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{131}); err != nil { + return err + } + + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { + return err + } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } + + // t.t.Allocated (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Allocated))); err != nil { + return err + } + + // t.t.Committed (bool) (bool) + if err := cbg.WriteBool(w, t.Committed); err != nil { + return err + } + return nil +} + +func (t *DealMapping) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + // t.t.Allocated (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Allocated = uint64(extra) + // t.t.Committed (bool) (bool) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Committed = false + case 21: + t.Committed = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + return nil +} diff --git a/storage/sector/store.go b/storage/sector/store.go index 52c45deef..8b7f59d5c 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -1,33 +1,32 @@ package sector import ( + "bytes" "context" "fmt" "io" + "math/bits" "sync" "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func init() { - cbor.RegisterCborType(dealMapping{}) -} - var log = logging.Logger("sectorstore") var sectorDealsPrefix = datastore.NewKey("/sectordeals") -type dealMapping struct { +type DealMapping struct { DealIDs []uint64 + Allocated uint64 Committed bool } @@ -70,10 +69,10 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 k := datastore.NewKey(fmt.Sprint(sectorID)) e, err := s.deals.Get(k) - var deals dealMapping + var deals DealMapping switch err { case nil: - if err := cbor.DecodeInto(e, &deals); err != nil { + if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { return 0, err } if deals.Committed { @@ -82,7 +81,9 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 fallthrough case datastore.ErrNotFound: deals.DealIDs = append(deals.DealIDs, dealIDs...) - d, err := cbor.DumpObject(&deals) + deals.Allocated += size + + d, err := cborrpc.Dump(&deals) if err != nil { return 0, err } @@ -96,7 +97,40 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 return sectorID, nil } -func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { +func (s *Store) PieceSizesToFill(sectorID uint64) ([]uint64, error) { + s.dealsLk.Lock() + defer s.dealsLk.Unlock() + + k := datastore.NewKey(fmt.Sprint(sectorID)) + e, err := s.deals.Get(k) + if err != nil { + return nil, err + } + var info DealMapping + if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &info); err != nil { + return nil, err + } + if info.Allocated > s.sb.SectorSize() { + return nil, xerrors.Errorf("more data allocated in sector than should be able to fit: %d > %d", info.Allocated, s.sb.SectorSize()) + } + + return fillersFromRem(sectorbuilder.UserBytesForSectorSize(s.sb.SectorSize()) - info.Allocated) +} + +func fillersFromRem(toFill uint64) ([]uint64, error) { + toFill += toFill / 127 // convert to in-sector bytes for easier math + + out := make([]uint64, bits.OnesCount64(toFill)) + for i := range out { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << next + toFill ^= psize + out[i] = sectorbuilder.UserBytesForSectorSize(psize) + } + return out, nil +} + +func (s *Store) DealsForCommit(sectorID uint64, commit bool) ([]uint64, error) { s.dealsLk.Lock() defer s.dealsLk.Unlock() @@ -105,16 +139,20 @@ func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) { switch err { case nil: - var deals dealMapping - if err := cbor.DecodeInto(e, &deals); err != nil { + var deals DealMapping + if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { return nil, err } + if !commit { + return nil, nil + } + if deals.Committed { log.Errorf("getting deal IDs for sector %d: sector already marked as committed", sectorID) } deals.Committed = true - d, err := cbor.DumpObject(&deals) + d, err := cborrpc.Dump(&deals) if err != nil { return nil, err } diff --git a/storage/sector/store_test.go b/storage/sector/store_test.go index a2e1d00ac..613f2836b 100644 --- a/storage/sector/store_test.go +++ b/storage/sector/store_test.go @@ -1,12 +1,46 @@ package sector import ( - "gotest.tools/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) -func TestComputePaddedSize(t *testing.T) { - assert.Equal(t, uint64(1040384), computePaddedSize(1000000)) - assert.Equal(t, uint64(1016), computePaddedSize(548)) - assert.Equal(t, uint64(4064), computePaddedSize(2048)) +func testFill(t *testing.T, n uint64, exp []uint64) { + f, err := fillersFromRem(n) + assert.NoError(t, err) + assert.Equal(t, exp, f) + + var sum uint64 + for _, u := range f { + sum += u + } + assert.Equal(t, n, sum) +} + +func TestFillersFromRem(t *testing.T) { + for i := 8; i < 32; i++ { + // single + ub := sectorbuilder.UserBytesForSectorSize(uint64(1) << i) + testFill(t, ub, []uint64{ub}) + + // 2 + ub = sectorbuilder.UserBytesForSectorSize(uint64(5) << i) + ub1 := sectorbuilder.UserBytesForSectorSize(uint64(1) << i) + ub3 := sectorbuilder.UserBytesForSectorSize(uint64(4) << i) + testFill(t, ub, []uint64{ub1, ub3}) + + // 4 + ub = sectorbuilder.UserBytesForSectorSize(uint64(15) << i) + ub2 := sectorbuilder.UserBytesForSectorSize(uint64(2) << i) + ub4 := sectorbuilder.UserBytesForSectorSize(uint64(8) << i) + testFill(t, ub, []uint64{ub1, ub2, ub3, ub4}) + + // different 2 + ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i) + testFill(t, ub, []uint64{ub1, ub4}) + } + } diff --git a/storage/sector_states.go b/storage/sector_states.go index 1122ea9a6..b57a5e315 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -33,6 +33,32 @@ func (m *Miner) handle(ctx context.Context, sector SectorInfo, cb providerHandle }() } +func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { + log.Infow("performing filling up rest of the sector...", "sector", sector.SectorID) + + fillerSizes, err := m.secst.PieceSizesToFill(sector.SectorID) + if err != nil { + return nil, err + } + + if len(fillerSizes) > 0 { + log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID) + } + + ids, err := m.storeGarbage(ctx, fillerSizes...) + if err != nil { + return nil, xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) + } + + for _, id := range ids { + if id != sector.SectorID { + panic("todo: pass SectorID into storeGarbage") + } + } + + return nil, nil +} + func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { log.Infow("performing sector replication...", "sector", sector.SectorID) sinfo, err := m.secst.SealPreCommit(ctx, sector.SectorID) @@ -135,7 +161,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector return nil, xerrors.Errorf("computing seal proof failed: %w", err) } - deals, err := m.secst.DealsForCommit(sector.SectorID) + deals, err := m.secst.DealsForCommit(sector.SectorID, true) if err != nil { return nil, err } From d615b7242a1f9a987838d1967aa630fd44c75c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 00:19:50 +0100 Subject: [PATCH 152/230] mod tidy --- go.sum | 132 +-------------------------------------------------------- 1 file changed, 2 insertions(+), 130 deletions(-) diff --git a/go.sum b/go.sum index 5645c179e..d02fd9826 100644 --- a/go.sum +++ b/go.sum @@ -13,10 +13,8 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= @@ -30,8 +28,6 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= @@ -75,7 +71,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA= @@ -88,35 +83,16 @@ github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -126,29 +102,11 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg= -github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -163,7 +121,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -174,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -291,8 +247,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -300,13 +254,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -433,10 +382,8 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI= github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= -github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= @@ -445,8 +392,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -454,7 +401,6 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -466,17 +412,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= -github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -504,30 +442,22 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg= github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc= github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M= github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -542,32 +472,16 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= @@ -577,27 +491,17 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= -github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c= -github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -607,19 +511,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -677,13 +576,11 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -693,11 +590,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -711,7 +606,6 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -726,7 +620,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -739,7 +632,6 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -747,15 +639,11 @@ golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -764,26 +652,17 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -804,25 +683,22 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y= gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -836,7 +712,3 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= From e075eb163fea3ce88b1cc19a085884595bb7f8dc Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 17:22:16 -0800 Subject: [PATCH 153/230] Add a test for the sectorstore --- storage/sector/store_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/storage/sector/store_test.go b/storage/sector/store_test.go index 613f2836b..cf7fe5e0b 100644 --- a/storage/sector/store_test.go +++ b/storage/sector/store_test.go @@ -1,11 +1,17 @@ package sector import ( + "context" + "fmt" + "io" + "math/rand" "testing" "github.com/stretchr/testify/assert" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/ipfs/go-datastore" ) func testFill(t *testing.T, n uint64, exp []uint64) { @@ -44,3 +50,33 @@ func TestFillersFromRem(t *testing.T) { } } + +func TestSectorStore(t *testing.T) { + if err := build.GetParams(true); err != nil { + t.Fatal(err) + } + + sb, cleanup, err := sectorbuilder.TempSectorbuilder(1024) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + tktFn := func(context.Context) (*sectorbuilder.SealTicket, error) { + return §orbuilder.SealTicket{ + BlockHeight: 17, + TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, + }, nil + } + + ds := datastore.NewMapDatastore() + + store := NewStore(sb, ds, tktFn) + + pr := io.LimitReader(rand.New(rand.NewSource(17)), 300) + sid, err := store.AddPiece("a", 300, pr, 1) + if err != nil { + t.Fatal(err) + } + fmt.Println(sid) +} From 9863942fe231b3217d00475676145ad499107a8b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 5 Nov 2019 22:26:50 -0800 Subject: [PATCH 154/230] WIP: trying to write a test to reproduce the storage deal error --- chain/deals/client.go | 2 +- cmd/lotus-storage-miner/run.go | 3 +-- lib/sectorbuilder/sectorbuilder_test.go | 2 +- node/impl/client/client.go | 23 ++++++++++++++++++----- node/modules/testing/genesis.go | 4 ++-- node/node_test.go | 19 +++++++++++++------ storage/miner.go | 3 ++- storage/post.go | 3 ++- storage/sector/store.go | 20 ++++++++++++++++++-- storage/sectorblocks/blocks.go | 9 +-------- 10 files changed, 59 insertions(+), 29 deletions(-) diff --git a/chain/deals/client.go b/chain/deals/client.go index b99422e11..a64a6abcc 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/impl/full" @@ -226,7 +227,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID) if err != nil { - s.Reset() return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err) } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 1b2a53a80..4142233fd 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -8,13 +8,12 @@ import ( "os/signal" "syscall" - "github.com/filecoin-project/lotus/build" - "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 5ce934f04..28bbc4ebf 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -12,7 +12,7 @@ import ( const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - t.Skip("this is slow") + //t.Skip("this is slow") //os.Setenv("BELLMAN_NO_GPU", "1") build.SectorSizes = []uint64{sectorSize} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 17ab26731..7128cbb22 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -58,7 +58,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A // TODO: make this a param self, err := a.WalletDefaultAddress(ctx) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to get default address: %w", err) } // get miner peerID @@ -70,11 +70,15 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A r, err := a.StateCall(ctx, msg, nil) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed getting peer ID: %w", err) } + if r.ExitCode != 0 { + return nil, xerrors.Errorf("call to get peer ID for miner failed: exit code %d", r.ExitCode) + } + pid, err := peer.IDFromBytes(r.Return) if err != nil { - return nil, err + return nil, xerrors.Errorf("parsing peer ID wrong: %w", err) } proposal := deals.ClientDealProposal{ @@ -88,7 +92,12 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A } c, err := a.DealClient.Start(ctx, proposal) - return &c, err + // TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!) + if err != nil { + return nil, xerrors.Errorf("failed to start deal: %w", err) + } + + return &c, nil } func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { @@ -177,7 +186,11 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) { return cid.Undef, err } - return nd.Cid(), bufferedDS.Commit() + if err := bufferedDS.Commit(); err != nil { + return cid.Undef, err + } + + return nd.Cid(), nil } func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) { diff --git a/node/modules/testing/genesis.go b/node/modules/testing/genesis.go index b53e33319..6c81f68ba 100644 --- a/node/modules/testing/genesis.go +++ b/node/modules/testing/genesis.go @@ -25,7 +25,7 @@ import ( var glog = logging.Logger("genesis") -func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { +func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis { return func() (*types.BlockHeader, error) { glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network") @@ -38,7 +38,7 @@ func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wal gmc := &gen.GenMinerCfg{ Owners: []address.Address{w}, Workers: []address.Address{w}, - PeerIDs: []peer.ID{"peerID 1"}, + PeerIDs: []peer.ID{minerPid}, } alloc := map[address.Address]types.BigInt{ w: types.FromFil(10000), diff --git a/node/node_test.go b/node/node_test.go index b035a7926..74833227b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -31,15 +31,12 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode { +func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode) test.TestStorageNode { r := repo.NewMemory(nil) lr, err := r.Lock(repo.RepoStorageMiner) require.NoError(t, err) - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) - ks, err := lr.KeyStore() require.NoError(t, err) @@ -115,12 +112,18 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te fulls := make([]test.TestNode, nFull) storers := make([]test.TestStorageNode, len(storage)) + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + minerPid, err := peer.IDFromPrivateKey(pk) + require.NoError(t, err) + var genbuf bytes.Buffer for i := 0; i < nFull; i++ { var genesis node.Option if i == 0 { - genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf)) + genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid)) } else { genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes())) } @@ -171,7 +174,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te genMiner, err := address.NewFromString("t0101") require.NoError(t, err) - storers[i] = testStorageNode(ctx, t, wa, genMiner, f) + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f) } if err := mn.LinkAll(); err != nil { @@ -221,3 +224,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test func TestAPIRPC(t *testing.T) { test.TestApis(t, rpcBuilder) } + +func TestAPIDealFlow(t *testing.T) { + test.TestDealFlow(t, builder) +} diff --git a/storage/miner.go b/storage/miner.go index ae93e78f5..72a6a0a8b 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,9 +2,10 @@ package storage import ( "context" + "sync" + "github.com/filecoin-project/lotus/lib/statestore" "github.com/ipfs/go-datastore/namespace" - "sync" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" diff --git a/storage/post.go b/storage/post.go index 3fe45d24c..ff21c48f7 100644 --- a/storage/post.go +++ b/storage/post.go @@ -2,9 +2,10 @@ package storage import ( "context" + "time" + "github.com/ipfs/go-cid" "go.opencensus.io/trace" - "time" "golang.org/x/xerrors" diff --git a/storage/sector/store.go b/storage/sector/store.go index 8b7f59d5c..453bb1c64 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -7,6 +7,7 @@ import ( "io" "math/bits" "sync" + "testing/iotest" "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-datastore" @@ -59,9 +60,24 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er } func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - sectorID, err = s.sb.AddPiece(ref, size, r) + padSize := computePaddedSize(size) + + buf := make([]byte, padSize) + r = iotest.NewReadLogger("UNIX FILE", r) + n, err := io.ReadFull(r, buf) if err != nil { - return 0, err + return 0, xerrors.Errorf("failed a bad thing: %w", err) + } + if uint64(n) != size { + panic("bad bad") + } + + bufr := bytes.NewReader(buf) + //r = io.MultiReader(r, io.LimitReader(nullReader{}, int64(padSize-size))) + + sectorID, err = s.sb.AddPiece(ref, padSize, bufr) + if err != nil { + return 0, xerrors.Errorf("sector store AddPiece call failed: %w", err) } s.dealsLk.Lock() diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 1d1289def..d8666b41c 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "errors" - "io" "sync" "github.com/ipfs/go-cid" @@ -20,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/cborrpc" - "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sector" @@ -132,9 +130,6 @@ func (r *refStorer) Read(p []byte) (n int, err error) { for { data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { - if err == io.EOF { - return 0, io.EOF - } return 0, xerrors.Errorf("reading block: %w", err) } @@ -173,9 +168,7 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6 intermediate: st.intermediate, } - pr, psize := padreader.New(r, uint64(size)) - - return st.Store.AddPiece(refst.pieceRef, psize, pr, dealID) + return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { From a49b949dca11e2d4cb784ba47f7b74d27876764c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 08:10:44 -0800 Subject: [PATCH 155/230] add file i didnt add before --- api/test/deals.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 api/test/deals.go diff --git a/api/test/deals.go b/api/test/deals.go new file mode 100644 index 000000000..1afab2759 --- /dev/null +++ b/api/test/deals.go @@ -0,0 +1,69 @@ +package test + +import ( + "context" + "fmt" + "io" + "math/rand" + "testing" + "time" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl" +) + +func TestDealFlow(t *testing.T, b APIBuilder) { + ctx := context.TODO() + n, sn := b(t, 1, []int{0}) + client := n[0].FullNode.(*impl.FullNodeAPI) + miner := sn[0] + _ = miner + + addrinfo, err := client.NetAddrsListen(ctx) + if err != nil { + t.Fatal(err) + } + addrinfo.Addrs = nil + + if err := miner.NetConnect(ctx, addrinfo); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + + r := io.LimitReader(rand.New(rand.NewSource(17)), 350) + fcid, err := client.ClientImportLocal(ctx, r) + if err != nil { + t.Fatal(err) + } + + maddr, err := address.NewFromString("t0101") + if err != nil { + t.Fatal(err) + } + + fmt.Println("FILE CID: ", fcid) + + go func() { + time.Sleep(time.Second) + fmt.Println("mining a block now") + if err := n[0].MineOne(ctx); err != nil { + t.Fatal(err) + } + fmt.Println("mined a block") + + time.Sleep(time.Second) + fmt.Println("mining a block now") + if err := n[0].MineOne(ctx); err != nil { + t.Fatal(err) + } + + }() + deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(1), 100) + if err != nil { + t.Fatal(err) + } + fmt.Println("Deal done!", deal) + + time.Sleep(time.Second * 10) +} From 7cb4148b186301eaa033d9cdfd6ca929b5e2d608 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 09:21:37 -0800 Subject: [PATCH 156/230] storage miner node needs mocknet option --- api/test/deals.go | 1 - node/node_test.go | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 1afab2759..656986c91 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -24,7 +24,6 @@ func TestDealFlow(t *testing.T, b APIBuilder) { if err != nil { t.Fatal(err) } - addrinfo.Addrs = nil if err := miner.NetConnect(ctx, addrinfo); err != nil { t.Fatal(err) diff --git a/node/node_test.go b/node/node_test.go index 74833227b..1d8c20b03 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -31,7 +31,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" ) -func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode) test.TestStorageNode { +func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode { r := repo.NewMemory(nil) lr, err := r.Lock(repo.RepoStorageMiner) @@ -90,6 +90,8 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a node.Repo(r), node.Test(), + node.MockHost(mn), + node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)), node.Override(new(api.FullNode), tnd), ) @@ -174,7 +176,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te genMiner, err := address.NewFromString("t0101") require.NoError(t, err) - storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f) + storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn) } if err := mn.LinkAll(); err != nil { From 597dbe369cb3ac04de9b3963a97b3179b29f3a9e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 11:44:28 -0800 Subject: [PATCH 157/230] Add method to query latest deal state --- api/api_full.go | 2 ++ api/struct.go | 4 ++++ api/test/deals.go | 35 +++++++++++++++++++++-------------- chain/deals/client.go | 8 ++++++++ lib/statestore/store.go | 14 ++++++++++++++ node/impl/client/client.go | 16 ++++++++++++++++ 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index e071a7066..bab1002ab 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -2,6 +2,7 @@ package api import ( "context" + "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" "github.com/libp2p/go-libp2p-core/peer" @@ -73,6 +74,7 @@ type FullNode interface { // ClientImport imports file under the specified path into filestore ClientImport(ctx context.Context, path string) (cid.Cid, error) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error) + ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error) ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) diff --git a/api/struct.go b/api/struct.go index 6eefe41dc..437e37624 100644 --- a/api/struct.go +++ b/api/struct.go @@ -82,6 +82,7 @@ type FullNodeStruct struct { ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"` ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"` ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` @@ -193,6 +194,9 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]Qu func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } +func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) { + return c.Internal.ClientGetDealInfo(ctx, deal) +} func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) { return c.Internal.ClientListDeals(ctx) diff --git a/api/test/deals.go b/api/test/deals.go index 656986c91..6db2f7331 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -8,17 +8,19 @@ import ( "testing" "time" + logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl" ) func TestDealFlow(t *testing.T, b APIBuilder) { + logging.SetAllLoggers(logging.LevelInfo) ctx := context.TODO() n, sn := b(t, 1, []int{0}) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] - _ = miner addrinfo, err := client.NetAddrsListen(ctx) if err != nil { @@ -44,24 +46,29 @@ func TestDealFlow(t *testing.T, b APIBuilder) { fmt.Println("FILE CID: ", fcid) go func() { - time.Sleep(time.Second) - fmt.Println("mining a block now") - if err := n[0].MineOne(ctx); err != nil { - t.Fatal(err) + for i := 0; i < 4; i++ { + time.Sleep(time.Second) + fmt.Println("mining a block now", i) + if err := n[0].MineOne(ctx); err != nil { + t.Fatal(err) + } } - fmt.Println("mined a block") - - time.Sleep(time.Second) - fmt.Println("mining a block now") - if err := n[0].MineOne(ctx); err != nil { - t.Fatal(err) - } - }() - deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(1), 100) + deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(200), 100) if err != nil { t.Fatal(err) } + + // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this + time.Sleep(time.Second) + for { + di, err := client.ClientGetDealInfo(ctx, *deal) + if err != nil { + t.Fatal(err) + } + fmt.Println("DEAL STATE: ", *deal, di.State) + time.Sleep(time.Second / 2) + } fmt.Println("Deal done!", deal) time.Sleep(time.Second * 10) diff --git a/chain/deals/client.go b/chain/deals/client.go index a64a6abcc..3fb982ecc 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -298,6 +298,14 @@ func (c *Client) List() ([]ClientDeal, error) { return out, nil } +func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) { + var out ClientDeal + if err := c.deals.Get(d, &out); err != nil { + return nil, err + } + return &out, nil +} + func (c *Client) Stop() { close(c.stop) <-c.stopped diff --git a/lib/statestore/store.go b/lib/statestore/store.go index ca924d66e..80df4a078 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -7,6 +7,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" + cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/lib/cborrpc" @@ -110,6 +111,19 @@ func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error) return st.ds.Put(k, mutated) } +func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error { + k := toKey(i) + val, err := st.ds.Get(k) + if err != nil { + if xerrors.Is(err, datastore.ErrNotFound) { + return xerrors.Errorf("No state for %s", i) + } + return err + } + + return out.UnmarshalCBOR(bytes.NewReader(val)) +} + // out: *[]T func (st *StateStore) List(out interface{}) error { res, err := st.ds.Query(query.Query{}) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7128cbb22..e4d8bd5d5 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -124,6 +124,22 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return out, nil } +func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) { + v, err := a.DealClient.GetDeal(d) + if err != nil { + return nil, err + } + return &api.DealInfo{ + ProposalCid: v.ProposalCid, + State: v.State, + Provider: v.Proposal.Provider, + PieceRef: v.Proposal.PieceRef, + Size: v.Proposal.PieceSize, + PricePerEpoch: v.Proposal.StoragePricePerEpoch, + Duration: v.Proposal.Duration, + }, nil +} + func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) { // TODO: check if we have the ENTIRE dag From 41d13b12a037a4d58b44c3d3f885f277e5600125 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 12:39:07 -0800 Subject: [PATCH 158/230] test is getting close to passing --- api/test/deals.go | 16 +++++++++++----- storage/sector/store.go | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 6db2f7331..4b36daa09 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -10,6 +10,7 @@ import ( logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl" @@ -54,7 +55,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { } } }() - deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(200), 100) + deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(400), 100) if err != nil { t.Fatal(err) } @@ -66,10 +67,15 @@ func TestDealFlow(t *testing.T, b APIBuilder) { if err != nil { t.Fatal(err) } - fmt.Println("DEAL STATE: ", *deal, di.State) + switch di.State { + case api.DealRejected: + t.Fatal("deal rejected") + case api.DealFailed: + t.Fatal("deal failed") + case api.DealComplete: + fmt.Println("COMPLETE", di) + break + } time.Sleep(time.Second / 2) } - fmt.Println("Deal done!", deal) - - time.Sleep(time.Second * 10) } diff --git a/storage/sector/store.go b/storage/sector/store.go index 453bb1c64..a3ae60544 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -59,6 +59,25 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er return &status, nil } +func computePaddedSize(size uint64) uint64 { + logv := 64 - bits.LeadingZeros64(size) + sectSize := uint64(1 << logv) + bound := sectorbuilder.UserBytesForSectorSize(sectSize) + if size <= bound { + return bound + } + return sectorbuilder.UserBytesForSectorSize(1 << (logv + 1)) +} + +type nullReader struct{} + +func (nr nullReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { padSize := computePaddedSize(size) From 480e7899ef8ffb5e4563860a30a7b6da0a8d574f Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 12:54:40 -0800 Subject: [PATCH 159/230] fail test if deal errors --- api/test/deals.go | 3 +++ lib/sectorbuilder/sectorbuilder_test.go | 2 +- node/impl/client/client.go | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 4b36daa09..ea5956458 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -72,10 +72,13 @@ func TestDealFlow(t *testing.T, b APIBuilder) { t.Fatal("deal rejected") case api.DealFailed: t.Fatal("deal failed") + case api.DealError: + t.Fatal("deal errored") case api.DealComplete: fmt.Println("COMPLETE", di) break } + fmt.Println("Deal state: ", di.State) time.Sleep(time.Second / 2) } } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 28bbc4ebf..5ce934f04 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -12,7 +12,7 @@ import ( const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - //t.Skip("this is slow") + t.Skip("this is slow") //os.Setenv("BELLMAN_NO_GPU", "1") build.SectorSizes = []uint64{sectorSize} diff --git a/node/impl/client/client.go b/node/impl/client/client.go index e4d8bd5d5..8856452f1 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -92,7 +92,6 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A } c, err := a.DealClient.Start(ctx, proposal) - // TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!) if err != nil { return nil, xerrors.Errorf("failed to start deal: %w", err) } From 74ea4d71c5a9f48c1655a213f52e2044808285ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 00:47:48 +0100 Subject: [PATCH 160/230] post-rebase fixes --- storage/sector/store.go | 46 ++++------------------------------ storage/sectorblocks/blocks.go | 9 ++++++- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/storage/sector/store.go b/storage/sector/store.go index a3ae60544..7f1159090 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -4,16 +4,14 @@ import ( "bytes" "context" "fmt" - "io" - "math/bits" - "sync" - "testing/iotest" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log" "golang.org/x/xerrors" + "io" + "math/bits" + "sync" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/cborrpc" @@ -59,44 +57,10 @@ func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, er return &status, nil } -func computePaddedSize(size uint64) uint64 { - logv := 64 - bits.LeadingZeros64(size) - sectSize := uint64(1 << logv) - bound := sectorbuilder.UserBytesForSectorSize(sectSize) - if size <= bound { - return bound - } - return sectorbuilder.UserBytesForSectorSize(1 << (logv + 1)) -} - -type nullReader struct{} - -func (nr nullReader) Read(b []byte) (int, error) { - for i := range b { - b[i] = 0 - } - return len(b), nil -} - func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - padSize := computePaddedSize(size) - - buf := make([]byte, padSize) - r = iotest.NewReadLogger("UNIX FILE", r) - n, err := io.ReadFull(r, buf) + sectorID, err = s.sb.AddPiece(ref, size, r) if err != nil { - return 0, xerrors.Errorf("failed a bad thing: %w", err) - } - if uint64(n) != size { - panic("bad bad") - } - - bufr := bytes.NewReader(buf) - //r = io.MultiReader(r, io.LimitReader(nullReader{}, int64(padSize-size))) - - sectorID, err = s.sb.AddPiece(ref, padSize, bufr) - if err != nil { - return 0, xerrors.Errorf("sector store AddPiece call failed: %w", err) + return 0, err } s.dealsLk.Lock() diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index d8666b41c..1d1289def 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "io" "sync" "github.com/ipfs/go-cid" @@ -19,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sector" @@ -130,6 +132,9 @@ func (r *refStorer) Read(p []byte) (n int, err error) { for { data, offset, nd, err := r.blockReader.ReadBlock(context.TODO()) if err != nil { + if err == io.EOF { + return 0, io.EOF + } return 0, xerrors.Errorf("reading block: %w", err) } @@ -168,7 +173,9 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6 intermediate: st.intermediate, } - return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID) + pr, psize := padreader.New(r, uint64(size)) + + return st.Store.AddPiece(refst.pieceRef, psize, pr, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { From cfac6f32728c56e2a34df549aea6bc9f64fc4df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 00:50:37 +0100 Subject: [PATCH 161/230] Fix TestSectorStore --- storage/sector/store_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/sector/store_test.go b/storage/sector/store_test.go index cf7fe5e0b..78d7ccd0e 100644 --- a/storage/sector/store_test.go +++ b/storage/sector/store_test.go @@ -3,6 +3,7 @@ package sector import ( "context" "fmt" + "github.com/filecoin-project/lotus/lib/padreader" "io" "math/rand" "testing" @@ -74,7 +75,9 @@ func TestSectorStore(t *testing.T) { store := NewStore(sb, ds, tktFn) pr := io.LimitReader(rand.New(rand.NewSource(17)), 300) - sid, err := store.AddPiece("a", 300, pr, 1) + pr, n := padreader.New(pr, 300) + + sid, err := store.AddPiece("a", n, pr, 1) if err != nil { t.Fatal(err) } From d45544c90c92ee1c6b6f3a6d69924b1c70ba89fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 00:59:56 +0100 Subject: [PATCH 162/230] Correct proposal cid on provider side --- chain/deals/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 46726298c..f54b32cf5 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -193,7 +193,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { - proposalNd, err := cborrpc.AsIpld(&proposal) + proposalNd, err := cborrpc.AsIpld(proposal.DealProposal) if err != nil { return MinerDeal{}, err } @@ -222,7 +222,7 @@ func (p *Provider) HandleStream(s inet.Stream) { deal, err := p.newDeal(s, proposal) if err != nil { - log.Error(err) + log.Errorf("%+v", err) s.Close() return } From 29778cd6533ede92bd9fae9ee0f212a87de5a55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 01:18:06 +0100 Subject: [PATCH 163/230] Mostly fix TestDealFlow --- api/test/deals.go | 20 ++++++++++++++++---- api/types.go | 11 +++++++++++ chain/sub/incoming.go | 2 +- chain/sync.go | 4 ++-- miner/miner.go | 2 +- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index ea5956458..050f1d0ee 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "math/rand" + "os" "testing" "time" @@ -17,6 +18,8 @@ import ( ) func TestDealFlow(t *testing.T, b APIBuilder) { + os.Setenv("BELLMAN_NO_GPU", "1") + logging.SetAllLoggers(logging.LevelInfo) ctx := context.TODO() n, sn := b(t, 1, []int{0}) @@ -46,10 +49,14 @@ func TestDealFlow(t *testing.T, b APIBuilder) { fmt.Println("FILE CID: ", fcid) + mine := true + done := make(chan struct{}) + go func() { - for i := 0; i < 4; i++ { + defer close(done) + for mine { time.Sleep(time.Second) - fmt.Println("mining a block now", i) + fmt.Println("mining a block now") if err := n[0].MineOne(ctx); err != nil { t.Fatal(err) } @@ -62,6 +69,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) + loop: for { di, err := client.ClientGetDealInfo(ctx, *deal) if err != nil { @@ -76,9 +84,13 @@ func TestDealFlow(t *testing.T, b APIBuilder) { t.Fatal("deal errored") case api.DealComplete: fmt.Println("COMPLETE", di) - break + break loop } - fmt.Println("Deal state: ", di.State) + fmt.Println("Deal state: ", api.DealStates[di.State]) time.Sleep(time.Second / 2) } + + mine = false + fmt.Println("shutting down mining") + <-done } diff --git a/api/types.go b/api/types.go index 6a5153328..cd2717bbd 100644 --- a/api/types.go +++ b/api/types.go @@ -25,6 +25,17 @@ const ( DealNoUpdate = DealUnknown ) +var DealStates = []string{ + "DealUnknown", + "DealRejected", + "DealAccepted", + "DealStaged", + "DealSealing", + "DealFailed", + "DealComplete", + "DealError", +} + // TODO: check if this exists anywhere else type MultiaddrSlice []ma.Multiaddr diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index ddca848e9..5e6c31de1 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -44,7 +44,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha return } - log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom()) + log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom()) s.InformNewBlock(msg.GetFrom(), &types.FullBlock{ Header: blk.Header, BlsMessages: bmsgs, diff --git a/chain/sync.go b/chain/sync.go index dac2a79e5..952b57a20 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -102,7 +102,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { if from == syncer.self { // TODO: this is kindof a hack... - log.Info("got block from ourselves") + log.Debug("got block from ourselves") if err := syncer.Sync(ctx, fts.TipSet()); err != nil { log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err) @@ -907,7 +907,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error } syncer.syncState.SetStage(api.StageSyncComplete) - log.Infow("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids())) + log.Debugw("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids())) return nil } diff --git a/miner/miner.go b/miner/miner.go index 9f7c44478..72ec090c7 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -230,7 +230,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) } func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) { - log.Infow("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids())) + log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids())) ticket, err := m.scratchTicket(ctx, base) if err != nil { return nil, errors.Wrap(err, "scratching ticket failed") From 69d7a9910e28aec122913cfc3f4169455eb2796b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 01:32:10 +0100 Subject: [PATCH 164/230] deals: remove outdated comment --- chain/deals/provider_states.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 5e5dbc273..96279d7fb 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -256,9 +256,6 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal if err != nil { return nil, err }*/ - // TODO: Spec doesn't say anything about inclusion proofs anywhere - // Not sure what mechanisms prevents miner from storing data that isn't - // clients' data return nil, nil } From 54722a0d3834a965b905b316b88bec6c00602a83 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 6 Nov 2019 23:57:10 -0800 Subject: [PATCH 165/230] WIP: fixing the tests by waiting for chain activity --- api/api_full.go | 1 + api/struct.go | 5 ++++ chain/deals/client.go | 3 +++ chain/deals/client_states.go | 46 ++++++++++++++++++++++++++++++++++ chain/deals/provider_states.go | 1 + chain/stmgr/utils.go | 20 +++++++++++++++ node/impl/full/state.go | 7 +++++- storage/sector_states.go | 3 +++ 8 files changed, 85 insertions(+), 1 deletion(-) diff --git a/api/api_full.go b/api/api_full.go index bab1002ab..27c07e65e 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -109,6 +109,7 @@ type FullNode interface { StateMarketBalance(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) StateMarketParticipants(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) + StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) diff --git a/api/struct.go b/api/struct.go index 437e37624..682e8f3e7 100644 --- a/api/struct.go +++ b/api/struct.go @@ -105,6 +105,7 @@ type FullNodeStruct struct { StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"` StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` + StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` @@ -413,6 +414,10 @@ func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, ts *types.TipSet) return c.Internal.StateMarketDeals(ctx, ts) } +func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + return c.Internal.StateMarketStorageDeal(ctx, dealid, ts) +} + func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } diff --git a/chain/deals/client.go b/chain/deals/client.go index 3fb982ecc..4f9fdecc8 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" @@ -34,6 +35,7 @@ type ClientDeal struct { Proposal actors.StorageDealProposal State api.DealState Miner peer.ID + DealID uint64 s inet.Stream } @@ -46,6 +48,7 @@ type Client struct { dag dtypes.ClientDAG discovery *discovery.Local mpool full.MpoolAPI + events *events.Events deals *statestore.StateStore conns map[cid.Cid]inet.Stream diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index bb30e8440..982c68090 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" ) type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error @@ -126,6 +127,51 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) error { } func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { + //func (e *calledEvents) Called(check CheckFunc, hnd CalledHandler, rev RevertHandler, confidence int, timeout uint64, actor address.Address, method uint64) error { + checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { + sd, err := stmgr.GetStorageDeal(ctx, c.stmgr, deal.DealID, ts) + if err != nil { + return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch > 0 { + // Deal is active already! + panic("handle me") + return true, false, nil + } + + return false, true, nil + } + + called := func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + // To ask Magik: Does this trigger when the message in question is part of the parent state execution? Or just when its included in the block (aka, not executed) + // main thing i want to ensure is that ts.ParentState is the result of the execution of msg + + if msg == nil { + log.Error("timed out waiting for deal activation... what now?") + return false, nil + } + + // TODO: can check msg.Params to see if we should even bother checking the state + + sd, err := stmgr.GetStorageDeal(ctx, c.stmgr, deal.DealID, ts) + if err != nil { + return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + } + + if sd.ActivationEpoch == 0 { + return true, nil + } + + // Deal is active! + panic("handle me") + + return false, nil + } + + if err := c.events.Called(checkFunc, handler, rev, 3, 100, actors.StorageMarketAddress, actors.SMAMethods.ActivateStorageDeals); err != nil { + return xerrors.Errorf("failed to set up called handler") + } resp, err := c.readStorageDealResp(deal) if err != nil { return err diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 96279d7fb..a177bb1f1 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -247,6 +247,7 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal log.Warnf("Sending deal response failed: %s", err) } + log.Info("About to seal sector!", deal.ProposalCid, deal.SectorID) if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { return nil, xerrors.Errorf("sealing sector failed: %w", err) } diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 1dff70bbd..ecd5acf87 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -193,6 +193,26 @@ func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet, return minfo.SectorSize, nil } +func GetStorageDeal(ctx context.Context, sm *StateManager, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + var state actors.StorageMarketState + if _, err := sm.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil { + return nil, err + } + + blks := amt.WrapBlockstore(sm.ChainStore().Blockstore()) + da, err := amt.LoadAMT(blks, state.Deals) + if err != nil { + return nil, err + } + + var ocd actors.OnChainDeal + if err := da.Get(dealId, &ocd); err != nil { + return nil, err + } + + return &ocd, nil +} + func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) { blks := amt.WrapBlockstore(bs) a, err := amt.LoadAMT(blks, ssc) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 68a134246..5c9ee4050 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -3,9 +3,10 @@ package full import ( "bytes" "context" - "github.com/filecoin-project/go-amt-ipld" "strconv" + "github.com/filecoin-project/go-amt-ipld" + cid "github.com/ipfs/go-cid" "github.com/ipfs/go-hamt-ipld" "github.com/libp2p/go-libp2p-core/peer" @@ -279,3 +280,7 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[ } return out, nil } + +func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { + return stmgr.GetStorageDeal(ctx, s.StateManager, dealId, ts) +} diff --git a/storage/sector_states.go b/storage/sector_states.go index b57a5e315..a2662eb35 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" @@ -113,6 +114,7 @@ func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorI func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts + log.Info("Sector precommitted: ", sector.SectorID) mw, err := m.api.StateWaitMsg(ctx, *sector.PreCommitMessage) if err != nil { return nil, err @@ -122,6 +124,7 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect log.Error("sector precommit failed: ", mw.Receipt.ExitCode) return nil, err } + log.Info("precommit message landed on chain: ", sector.SectorID) randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - 1 // -1 because of how the messages are applied log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight) From bdb1c877a91b625e417613047e4b0264f883bc7a Mon Sep 17 00:00:00 2001 From: wanghui Date: Thu, 7 Nov 2019 16:18:52 +0800 Subject: [PATCH 166/230] fix ask price typo --- cli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 7f9771a3d..f61be20f0 100644 --- a/cli/client.go +++ b/cli/client.go @@ -307,7 +307,7 @@ var clientQueryAskCmd = &cli.Command{ } fmt.Printf("Ask: %s\n", maddr) - fmt.Printf("Price per Byte: %s\n", types.FIL(ask.Ask.Price)) + fmt.Printf("Price per GigaByte: %s\n", types.FIL(ask.Ask.Price)) size := cctx.Int64("size") if size == 0 { From 6cc360e0f7f35aa9de30974b5e86e50feca9a5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 10:02:29 +0100 Subject: [PATCH 167/230] events: Execute Called handlers in the correct tipset --- chain/events/events_called.go | 14 +++++++---- chain/events/events_test.go | 46 +++++++++++++++++------------------ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/chain/events/events_called.go b/chain/events/events_called.go index c51d9bef6..94368214e 100644 --- a/chain/events/events_called.go +++ b/chain/events/events_called.go @@ -24,7 +24,7 @@ type triggerH = uint64 // `ts` is the tipset, in which the `msg` is included. // `curH`-`ts.Height` = `confidence` -type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) +type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) // CheckFunc is used for atomicity guarantees. If the condition the callbacks // wait for has already happened in tipset `ts` @@ -147,7 +147,11 @@ func (e *calledEvents) checkNewCalls(ts *types.TipSet) { func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, ts *types.TipSet) { trigger := e.triggers[triggerId] - triggerH := ts.Height() + uint64(trigger.confidence) + + // messages are not applied in the tipset they are included in + appliedH := ts.Height() + 1 + + triggerH := appliedH + uint64(trigger.confidence) byOrigH, ok := e.confQueue[triggerH] if !ok { @@ -155,13 +159,13 @@ func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, e.confQueue[triggerH] = byOrigH } - byOrigH[ts.Height()] = append(byOrigH[ts.Height()], &queuedEvent{ + byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{ trigger: triggerId, - h: ts.Height(), + h: appliedH, msg: msg, }) - e.revertQueue[ts.Height()] = append(e.revertQueue[ts.Height()], triggerH) // todo: dedupe? + e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH) } func (e *calledEvents) applyWithConfidence(ts *types.TipSet) { diff --git a/chain/events/events_test.go b/chain/events/events_test.go index d738edf86..27a3edf03 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -526,28 +526,28 @@ func TestCalled(t *testing.T) { // create additional block so we are above confidence threshold - fcs.advance(0, 1, nil) // H=9 (confidence=3, apply) + fcs.advance(0, 2, nil) // H=10 (confidence=3, apply) require.Equal(t, true, applied) require.Equal(t, false, reverted) applied = false - require.Equal(t, uint64(6), appliedTs.Height()) - require.Equal(t, "bafkqaajq", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(9), appliedH) + require.Equal(t, uint64(7), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(10), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(1), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // revert some blocks, keep the message - fcs.advance(3, 1, nil) // H=7 (confidence=1) + fcs.advance(3, 1, nil) // H=8 (confidence=1) require.Equal(t, false, applied) require.Equal(t, false, reverted) // revert the message - fcs.advance(2, 1, nil) // H=6, we reverted ts with the msg + fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg require.Equal(t, false, applied) require.Equal(t, true, reverted) @@ -561,7 +561,7 @@ func TestCalled(t *testing.T) { }, }) - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=7; H=10 (confidence=3) + fcs.advance(0, 5, map[int]cid.Cid{ // (confidence=3) 0: n2msg, }) @@ -569,16 +569,16 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - require.Equal(t, uint64(7), appliedTs.Height()) - require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(10), appliedH) + require.Equal(t, uint64(9), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(12), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(2), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // revert and apply at different height - fcs.advance(4, 5, map[int]cid.Cid{ // msg at H=8; H=11 (confidence=3) + fcs.advance(4, 6, map[int]cid.Cid{ // (confidence=3) 1: n2msg, }) @@ -590,16 +590,16 @@ func TestCalled(t *testing.T) { reverted = false applied = false - require.Equal(t, uint64(8), appliedTs.Height()) - require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String()) - require.Equal(t, uint64(11), appliedH) + require.Equal(t, uint64(11), appliedTs.Height()) + require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String()) + require.Equal(t, uint64(14), appliedH) require.Equal(t, t0123, appliedMsg.To) require.Equal(t, uint64(2), appliedMsg.Nonce) require.Equal(t, uint64(5), appliedMsg.Method) // call method again - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=12; H=15 + fcs.advance(0, 5, map[int]cid.Cid{ 0: n2msg, }) @@ -608,7 +608,7 @@ func TestCalled(t *testing.T) { applied = false // send and revert below confidence, then cross confidence - fcs.advance(0, 1, map[int]cid.Cid{ // msg at H=16; H=16 + fcs.advance(0, 2, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 3}, @@ -623,7 +623,7 @@ func TestCalled(t *testing.T) { // test timeout (it's set to 20 in the call to `events.Called` above) - fcs.advance(0, 6, nil) // H=25 + fcs.advance(0, 6, nil) require.Equal(t, false, applied) // not calling timeout as we received messages require.Equal(t, false, reverted) @@ -631,7 +631,7 @@ func TestCalled(t *testing.T) { // test unregistering with more more = false - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29 + fcs.advance(0, 5, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more @@ -643,7 +643,7 @@ func TestCalled(t *testing.T) { require.Equal(t, false, reverted) applied = false - fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29 + fcs.advance(0, 5, map[int]cid.Cid{ 0: fcs.fakeMsgs(fakeMsg{ bmsgs: []*types.Message{ {To: t0123, From: t0123, Method: 5, Nonce: 5}, @@ -765,10 +765,10 @@ func TestCalledOrder(t *testing.T) { switch at { case 0: require.Equal(t, uint64(1), msg.Nonce) - require.Equal(t, uint64(3), ts.Height()) + require.Equal(t, uint64(4), ts.Height()) case 1: require.Equal(t, uint64(2), msg.Nonce) - require.Equal(t, uint64(4), ts.Height()) + require.Equal(t, uint64(5), ts.Height()) default: t.Fatal("apply should only get called twice, at: ", at) } @@ -777,9 +777,9 @@ func TestCalledOrder(t *testing.T) { }, func(_ context.Context, ts *types.TipSet) error { switch at { case 2: - require.Equal(t, uint64(4), ts.Height()) + require.Equal(t, uint64(5), ts.Height()) case 3: - require.Equal(t, uint64(3), ts.Height()) + require.Equal(t, uint64(4), ts.Height()) default: t.Fatal("revert should only get called twice, at: ", at) } From 12161fc60704697eb80b4778db9d80c0d5a2ce90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 10:06:06 +0100 Subject: [PATCH 168/230] deals: Finish client sealing state handler --- api/test/deals.go | 2 +- chain/deals/client.go | 6 ++-- chain/deals/client_states.go | 59 +++++++++++++++++++++++------------- node/impl/full/state.go | 2 +- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 050f1d0ee..41dda8228 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -69,7 +69,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) - loop: +loop: for { di, err := client.ClientGetDealInfo(ctx, *deal) if err != nil { diff --git a/chain/deals/client.go b/chain/deals/client.go index 4f9fdecc8..ddc0efb8b 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -66,7 +66,7 @@ type clientDealUpdate struct { err error } -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI) *Client { +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI, chainapi full.ChainAPI) *Client { c := &Client{ sm: sm, chain: chain, @@ -75,6 +75,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * dag: dag, discovery: discovery, mpool: mpool, + events: events.NewEvents(context.TODO(), &chainapi), deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), conns: map[cid.Cid]inet.Stream{}, @@ -157,7 +158,8 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { case api.DealStaged: c.handle(ctx, deal, c.staged, api.DealSealing) case api.DealSealing: - c.handle(ctx, deal, c.sealing, api.DealComplete) + c.handle(ctx, deal, c.sealing, api.DealNoUpdate) + // TODO: DealComplete -> watch for faults, expiration, etc. } } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 982c68090..e1157ad1d 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -2,6 +2,7 @@ package deals import ( "context" + "github.com/filecoin-project/lotus/build" "golang.org/x/xerrors" @@ -19,6 +20,11 @@ func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFu if err != nil { next = api.DealError } + + if err == nil && next == api.DealNoUpdate { + return + } + select { case c.updated <- clientDealUpdate{ newState: next, @@ -127,16 +133,25 @@ func (c *Client) staged(ctx context.Context, deal ClientDeal) error { } func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { - //func (e *calledEvents) Called(check CheckFunc, hnd CalledHandler, rev RevertHandler, confidence int, timeout uint64, actor address.Address, method uint64) error { + + // TODO: disconnect + checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { - sd, err := stmgr.GetStorageDeal(ctx, c.stmgr, deal.DealID, ts) + sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) if err != nil { + // TODO: This may be fine for some errors return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) } if sd.ActivationEpoch > 0 { - // Deal is active already! - panic("handle me") + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + }: + case <-c.stop: + } + return true, false, nil } @@ -144,8 +159,7 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { } called := func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { - // To ask Magik: Does this trigger when the message in question is part of the parent state execution? Or just when its included in the block (aka, not executed) - // main thing i want to ensure is that ts.ParentState is the result of the execution of msg + // TODO: handle errors if msg == nil { log.Error("timed out waiting for deal activation... what now?") @@ -154,34 +168,37 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { // TODO: can check msg.Params to see if we should even bother checking the state - sd, err := stmgr.GetStorageDeal(ctx, c.stmgr, deal.DealID, ts) + sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) if err != nil { - return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err) + return false, xerrors.Errorf("failed to look up deal on chain: %w", err) } if sd.ActivationEpoch == 0 { return true, nil } - // Deal is active! - panic("handle me") + log.Info("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) + + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + }: + case <-c.stop: + } return false, nil } - if err := c.events.Called(checkFunc, handler, rev, 3, 100, actors.StorageMarketAddress, actors.SMAMethods.ActivateStorageDeals); err != nil { + revert := func(ctx context.Context, ts *types.TipSet) error { + log.Warn("deal activation reverted; TODO: actually handle this!") + // TODO: Just go back to DealSealing? + return nil + } + + if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, deal.Proposal.Provider, actors.MAMethods.ProveCommitSector); err != nil { return xerrors.Errorf("failed to set up called handler") } - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err - } - - if resp.State != api.DealComplete { - return xerrors.Errorf("deal wasn't complete (State=%d)", resp.State) - } - - // TODO: look for the commit message on chain, negotiate better payment vouchers log.Info("DEAL COMPLETE!!") return nil diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 5c9ee4050..606e3416a 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -282,5 +282,5 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[ } func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) { - return stmgr.GetStorageDeal(ctx, s.StateManager, dealId, ts) + return stmgr.GetStorageDeal(ctx, a.StateManager, dealId, ts) } From f221c05a334b12363aca181479a74073aed0e2a2 Mon Sep 17 00:00:00 2001 From: wanghui Date: Thu, 7 Nov 2019 18:46:11 +0800 Subject: [PATCH 169/230] add sector size info --- cmd/lotus-storage-miner/info.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 436fc4686..287112494 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -37,6 +37,13 @@ var infoCmd = &cli.Command{ fmt.Printf("Miner: %s\n", maddr) + // Sector size + size, err := api.StateMinerSectorSize(ctx, maddr, nil) + if err != nil { + return err + } + fmt.Printf("Sector Size(Byte): %d\n", types.NewInt(size)) + pow, err := api.StateMinerPower(ctx, maddr, nil) if err != nil { return err @@ -56,7 +63,6 @@ var infoCmd = &cli.Command{ fmt.Println("Failed Sectors:\t", sinfo.FailedCount) // TODO: grab actr state / info - // * Sector size // * Sealed sectors (count / bytes) // * Power return nil From 08ff772cf41cd7746b2a3c05992cef6e011e6ec7 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 7 Nov 2019 19:33:44 +0800 Subject: [PATCH 170/230] Update cmd/lotus-storage-miner/info.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit update sector size format Co-Authored-By: Łukasz Magiera --- cmd/lotus-storage-miner/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 287112494..05946f807 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -42,7 +42,7 @@ var infoCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("Sector Size(Byte): %d\n", types.NewInt(size)) + fmt.Printf("Sector Size: %dKiB\n", size / 1024) pow, err := api.StateMinerPower(ctx, maddr, nil) if err != nil { From be0d07e14303056c19fe1391ffda730664b8a687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 13:03:18 +0100 Subject: [PATCH 171/230] pass deal IDs in SectorPreCommitInfo --- api/test/deals.go | 2 +- chain/actors/actor_miner.go | 56 +++++---- chain/actors/cbor_gen.go | 232 +++++++++++++++--------------------- chain/deals/cbor_gen.go | 19 ++- gen/main.go | 2 +- storage/sector_states.go | 13 +- 6 files changed, 153 insertions(+), 171 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 41dda8228..3145f7c4a 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -21,7 +21,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { os.Setenv("BELLMAN_NO_GPU", "1") logging.SetAllLoggers(logging.LevelInfo) - ctx := context.TODO() + ctx := context.Background() n, sn := b(t, 1, []int{0}) client := n[0].FullNode.(*impl.FullNodeAPI) miner := sn[0] diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 963a469e9..608cbd5df 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -24,7 +24,7 @@ type StorageMinerActor struct{} type StorageMinerActorState struct { // PreCommittedSectors is the set of sectors that have been committed to but not // yet had their proofs submitted - PreCommittedSectors map[string]*UnprovenSector + PreCommittedSectors map[string]*PreCommittedSector // All sectors this miner has committed. Sectors cid.Cid @@ -94,11 +94,9 @@ type MinerInfo struct { // SubsectorCount } -type UnprovenSector struct { - CommD []byte - CommR []byte - SubmitHeight uint64 - TicketEpoch uint64 +type PreCommittedSector struct { + Info SectorPreCommitInfo + ReceivedEpoch uint64 } type StorageMinerConstructorParams struct { @@ -108,6 +106,14 @@ type StorageMinerConstructorParams struct { PeerID peer.ID } +type SectorPreCommitInfo struct { + SectorNumber uint64 + + CommR []byte // TODO: Spec says CID + SealEpoch uint64 + DealIDs []uint64 +} + type maMethods struct { Constructor uint64 PreCommitSector uint64 @@ -212,14 +218,6 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ return nil, nil } -type SectorPreCommitInfo struct { - CommD []byte // TODO: update proofs code - CommR []byte - - Epoch uint64 - SectorNumber uint64 -} - func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorPreCommitInfo) ([]byte, ActorError) { ctx := vmctx.Context() oldstate, self, err := loadState(vmctx) @@ -227,12 +225,12 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon return nil, err } - if params.Epoch >= vmctx.BlockHeight()+build.SealRandomnessLookback { - return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.Epoch, vmctx.BlockHeight()+build.SealRandomnessLookback) + if params.SealEpoch >= vmctx.BlockHeight()+build.SealRandomnessLookback { + return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.SealEpoch, vmctx.BlockHeight()+build.SealRandomnessLookback) } - if vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit { - return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.Epoch+build.SealRandomnessLookback) + if vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit { + return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback) } mi, err := loadMinerInfo(vmctx, self) @@ -262,11 +260,9 @@ func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMCon return nil, aerrors.New(4, "not enough collateral") } - self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &UnprovenSector{ - CommR: params.CommR, - CommD: params.CommD, - SubmitHeight: vmctx.BlockHeight(), - TicketEpoch: params.Epoch, + self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &PreCommittedSector{ + Info: *params, + ReceivedEpoch: vmctx.BlockHeight(), } nstate, err := vmctx.Storage().Put(self) @@ -309,7 +305,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.New(1, "no pre-commitment found for sector") } - if us.SubmitHeight+build.InteractivePoRepDelay > vmctx.BlockHeight() { + if us.ReceivedEpoch+build.InteractivePoRepDelay > vmctx.BlockHeight() { return nil, aerrors.New(2, "too early for proof submission") } @@ -318,12 +314,12 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC // TODO: ensure normalization to ID address maddr := vmctx.Message().To - ticket, err := vmctx.GetRandomness(us.TicketEpoch - build.SealRandomnessLookback) + ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback) if err != nil { return nil, aerrors.Wrap(err, "failed to get ticket randomness") } - seed, err := vmctx.GetRandomness(us.SubmitHeight + build.InteractivePoRepDelay) + seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay) if err != nil { return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment") } @@ -341,16 +337,16 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.Wrap(err, "failed to compute data commitment") } - if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { + if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { - return nil, aerrors.Newf(2, "bad proof! (t:%x; s:%x(%d); p:%x)", ticket, seed, us.SubmitHeight+build.InteractivePoRepDelay, params.Proof) + return nil, aerrors.Newf(2, "bad proof! (t:%x; s:%x(%d); p:%x)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, params.Proof) } // Note: There must exist a unique index in the miner's sector set for each // sector ID. The `faults`, `recovered`, and `done` parameters of the // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.CommR, us.CommD) + nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.Info.CommR, commD) if err != nil { return nil, err } @@ -581,6 +577,8 @@ func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID u return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node") } + // TODO: Spec says to use SealCommitment, and construct commD from deals each time, + // but that would make SubmitPoSt way, way more expensive if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "failed to set commitment in sector set") } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 073286eb3..7b5aef2fb 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -201,7 +201,7 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return err } - // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) (map) + // t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map) if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { return err } @@ -291,7 +291,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.PreCommittedSectors (map[string]*actors.UnprovenSector) (map) + // t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -304,7 +304,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("t.PreCommittedSectors: map too large") } - t.PreCommittedSectors = make(map[string]*UnprovenSector, extra) + t.PreCommittedSectors = make(map[string]*PreCommittedSector, extra) for i, l := 0, int(extra); i < l; i++ { @@ -319,7 +319,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { k = string(sval) } - var v *UnprovenSector + var v *PreCommittedSector { @@ -333,7 +333,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { return err } } else { - v = new(UnprovenSector) + v = new(PreCommittedSector) if err := v.UnmarshalCBOR(br); err != nil { return err } @@ -546,11 +546,8 @@ func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.CommD ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { - return err - } - if _, err := w.Write(t.CommD); err != nil { + // t.t.SectorNumber (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { return err } @@ -562,15 +559,20 @@ func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error { return err } - // t.t.Epoch (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Epoch))); err != nil { + // t.t.SealEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SealEpoch))); err != nil { return err } - // t.t.SectorNumber (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { + // t.t.DealIDs ([]uint64) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { return err } + for _, v := range t.DealIDs { + if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { + return err + } + } return nil } @@ -589,50 +591,6 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.CommD ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.CommD: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommD = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommD); err != nil { - return err - } - // t.t.CommR ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.CommR: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommR = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommR); err != nil { - return err - } - // t.t.Epoch (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Epoch = uint64(extra) // t.t.SectorNumber (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) @@ -643,78 +601,6 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.SectorNumber = uint64(extra) - return nil -} - -func (t *UnprovenSector) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{132}); err != nil { - return err - } - - // t.t.CommD ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { - return err - } - if _, err := w.Write(t.CommD); err != nil { - return err - } - - // t.t.CommR ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil { - return err - } - if _, err := w.Write(t.CommR); err != nil { - return err - } - - // t.t.SubmitHeight (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SubmitHeight))); err != nil { - return err - } - - // t.t.TicketEpoch (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TicketEpoch))); err != nil { - return err - } - return nil -} - -func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 4 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.t.CommD ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.CommD: array too large (%d)", extra) - } - - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommD = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommD); err != nil { - return err - } // t.t.CommR ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) @@ -732,7 +618,7 @@ func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } - // t.t.SubmitHeight (uint64) (uint64) + // t.t.SealEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -741,8 +627,86 @@ func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.SubmitHeight = uint64(extra) - // t.t.TicketEpoch (uint64) (uint64) + t.SealEpoch = uint64(extra) + // t.t.DealIDs ([]uint64) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.DealIDs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.DealIDs = make([]uint64, extra) + } + for i := 0; i < int(extra); i++ { + + maj, val, err := cbg.CborReadHeader(br) + if err != nil { + return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) + } + + if maj != cbg.MajUnsignedInt { + return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) + } + + t.DealIDs[i] = val + } + + return nil +} + +func (t *PreCommittedSector) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.Info (actors.SectorPreCommitInfo) (struct) + if err := t.Info.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ReceivedEpoch (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ReceivedEpoch))); err != nil { + return err + } + return nil +} + +func (t *PreCommittedSector) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Info (actors.SectorPreCommitInfo) (struct) + + { + + if err := t.Info.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ReceivedEpoch (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -751,7 +715,7 @@ func (t *UnprovenSector) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.TicketEpoch = uint64(extra) + t.ReceivedEpoch = uint64(extra) return nil } diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 08edb9371..585836a89 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -577,7 +577,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{132}); err != nil { + if _, err := w.Write([]byte{133}); err != nil { return err } @@ -604,6 +604,11 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { if _, err := w.Write([]byte(t.Miner)); err != nil { return err } + + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } return nil } @@ -618,7 +623,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 4 { + if extra != 5 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -663,6 +668,16 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { t.Miner = peer.ID(sval) } + // t.t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) return nil } diff --git a/gen/main.go b/gen/main.go index 4c0c2167a..efe1af965 100644 --- a/gen/main.go +++ b/gen/main.go @@ -91,7 +91,7 @@ func main() { actors.StorageMinerActorState{}, actors.StorageMinerConstructorParams{}, actors.SectorPreCommitInfo{}, - actors.UnprovenSector{}, + actors.PreCommittedSector{}, actors.MinerInfo{}, actors.SubmitPoStParams{}, actors.PaymentVerifyParams{}, diff --git a/storage/sector_states.go b/storage/sector_states.go index a2662eb35..b52402c4f 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -78,12 +78,17 @@ func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*Sec } func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { - params := &actors.SectorPreCommitInfo{ - CommD: sector.CommD, - CommR: sector.CommR, - Epoch: sector.Ticket.BlockHeight, + deals, err := m.secst.DealsForCommit(sector.SectorID, false) + if err != nil { + return nil, err + } + params := &actors.SectorPreCommitInfo{ SectorNumber: sector.SectorID, + + CommR: sector.CommR, + SealEpoch: sector.Ticket.BlockHeight, + DealIDs: deals, } enc, aerr := actors.SerializeParams(params) if aerr != nil { From 66f8e348a21f24977b932401cf3aefbc0cfd6a4f Mon Sep 17 00:00:00 2001 From: wanghui Date: Thu, 7 Nov 2019 20:05:03 +0800 Subject: [PATCH 172/230] update size and unit --- cmd/lotus-storage-miner/info.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 05946f807..3ce7c0ff9 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -38,11 +38,13 @@ var infoCmd = &cli.Command{ fmt.Printf("Miner: %s\n", maddr) // Sector size - size, err := api.StateMinerSectorSize(ctx, maddr, nil) + sizeByte, err := api.StateMinerSectorSize(ctx, maddr, nil) if err != nil { return err } - fmt.Printf("Sector Size: %dKiB\n", size / 1024) + + size, unit := getSizeAndUnit(sizeByte) + fmt.Printf("Sector Size: %g %s\n", size, unit) pow, err := api.StateMinerPower(ctx, maddr, nil) if err != nil { @@ -69,6 +71,18 @@ var infoCmd = &cli.Command{ }, } +var Units = []string{"B", "KiB", "MiB", "GiB"} + +func getSizeAndUnit(size uint64) (float64, string) { + i := 0 + unitSize := float64(size) + for unitSize >= 1024 && i < len(Units) - 1 { + unitSize = unitSize / 1024 + i++ + } + return unitSize, Units[i] +} + type SectorsInfo struct { TotalCount int SealingCount int From eca55019236b955beec94c28d665407ef49da0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 13:06:32 +0100 Subject: [PATCH 173/230] remove some outdated todos --- api/api_full.go | 2 +- chain/actors/actor_storagemarket.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 27c07e65e..d46fd25ba 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -77,7 +77,7 @@ type FullNode interface { ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error) ClientListDeals(ctx context.Context) ([]DealInfo, error) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) - ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now) + ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index cc2e7a96e..4e00e970a 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -436,7 +436,6 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } @@ -502,7 +501,6 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } @@ -602,7 +600,6 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals) if err != nil { - // TODO: kind of annoying that this can be caused by gas, otherwise could be fatal return nil, aerrors.HandleExternalError(err, "loading deals amt") } From 79c9fb719e0b4c893960d01314b6408becdb7279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 13:57:00 +0100 Subject: [PATCH 174/230] deals: disconnect early, rely on chain --- chain/deals/cbor_gen.go | 40 ++-------------------------------- chain/deals/client_states.go | 33 ++++------------------------ chain/deals/client_utils.go | 11 ++++++++++ chain/deals/provider_states.go | 26 ++++++---------------- chain/deals/provider_utils.go | 11 ++++++++++ chain/deals/types.go | 3 --- 6 files changed, 35 insertions(+), 89 deletions(-) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 585836a89..9bdf7163a 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -192,7 +192,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{133}); err != nil { return err } @@ -232,18 +232,6 @@ func (t *Response) MarshalCBOR(w io.Writer) error { } } - // t.t.CommitMessage (cid.Cid) (struct) - - if t.CommitMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) - } - } - return nil } @@ -258,7 +246,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 5 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -338,30 +326,6 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.PublishMessage = &c } - } - // t.t.CommitMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) - } - - t.CommitMessage = &c - } - } return nil } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index e1157ad1d..a4e87d862 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -41,6 +41,9 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) error { if err != nil { return err } + if err := c.disconnect(deal); err != nil { + return err + } if resp.State != api.DealAccepted { return xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) @@ -102,40 +105,12 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) error { } func (c *Client) staged(ctx context.Context, deal ClientDeal) error { - /* miner seals our data, hopefully */ - - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err - } - - if resp.State != api.DealSealing { - return xerrors.Errorf("deal wasn't sealed (State=%d)", resp.State) - } - - log.Info("DEAL SEALED!") - - // TODO: want? - /*ssize, err := stmgr.GetMinerSectorSize(ctx, c.sm, nil, deal.Proposal.MinerAddress) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) - } - - ok, err := sectorbuilder.VerifyPieceInclusionProof(ssize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements) - if err != nil { - return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err) - } - if !ok { - return xerrors.Errorf("verifying piece inclusion proof in staged deal %s failed", deal.ProposalCid) - }*/ + // wait return nil } func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { - - // TODO: disconnect - checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) if err != nil { diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 7cdd77192..7f77dbe0a 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -85,3 +85,14 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { return &resp.Response, nil } + +func (c *Client) disconnect(deal ClientDeal) error { + s, ok := c.conns[deal.ProposalCid] + if !ok { + return nil + } + + err := s.Close() + delete(c.conns, deal.ProposalCid) + return err +} diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index a177bb1f1..468d45f4a 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -171,15 +171,20 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) log.Info("fetching data for a deal") mcid := smsg.Cid() err = p.sendSignedResponse(&Response{ - State: api.DealAccepted, - Message: "", + State: api.DealAccepted, + Proposal: deal.ProposalCid, PublishMessage: &mcid, + StorageDeal: &storageDeal, }) if err != nil { return nil, err } + if err := p.disconnect(deal); err != nil { + log.Warnf("closing client connection: %+v", err) + } + return func(deal *MinerDeal) { deal.DealID = resp.DealIDs[0] }, merkledag.FetchGraph(ctx, deal.Ref, p.dag) @@ -188,14 +193,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // STAGED func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(&Response{ - State: api.DealStaged, - Proposal: deal.ProposalCid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - root, err := p.dag.Get(ctx, deal.Ref) if err != nil { return nil, xerrors.Errorf("failed to get file root for deal: %s", err) @@ -238,15 +235,6 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // SEALING func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - err := p.sendSignedResponse(&Response{ - State: api.DealSealing, - Proposal: deal.ProposalCid, - // TODO: Send sector ID - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - } - log.Info("About to seal sector!", deal.ProposalCid, deal.SectorID) if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { return nil, xerrors.Errorf("sealing sector failed: %w", err) diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 2ea5dc05e..0a881d111 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -100,6 +100,17 @@ func (p *Provider) sendSignedResponse(resp *Response) error { return err } +func (p *Provider) disconnect(deal MinerDeal) error { + s, ok := p.conns[deal.ProposalCid] + if !ok { + return nil + } + + err := s.Close() + delete(p.conns, deal.ProposalCid) + return err +} + func (p *Provider) getWorker(miner address.Address) (address.Address, error) { getworker := &types.Message{ To: miner, diff --git a/chain/deals/types.go b/chain/deals/types.go index 2177517ab..4318aad39 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -27,9 +27,6 @@ type Response struct { // DealAccepted StorageDeal *actors.StorageDeal PublishMessage *cid.Cid - - // DealComplete - CommitMessage *cid.Cid } // TODO: Do we actually need this to be signed? From 8ec37e87283be0d737688ec638401a6423c860ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 14:29:43 +0100 Subject: [PATCH 175/230] deals: cleanup client state machine --- chain/deals/cbor_gen.go | 41 +++++++++++++- chain/deals/client.go | 6 +++ chain/deals/client_states.go | 101 +++++++++++++++++++++-------------- lib/cborrpc/rpc.go | 12 +++++ 4 files changed, 117 insertions(+), 43 deletions(-) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index 9bdf7163a..c3be140ac 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -541,7 +541,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{133}); err != nil { + if _, err := w.Write([]byte{134}); err != nil { return err } @@ -573,6 +573,19 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err } + + // t.t.PublishMessage (cid.Cid) (struct) + + if t.PublishMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) + } + } + return nil } @@ -587,7 +600,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -642,6 +655,30 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) + // t.t.PublishMessage (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + } + + t.PublishMessage = &c + } + + } return nil } diff --git a/chain/deals/client.go b/chain/deals/client.go index ddc0efb8b..a07c90827 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -37,6 +37,8 @@ type ClientDeal struct { Miner peer.ID DealID uint64 + PublishMessage *cid.Cid + s inet.Stream } @@ -64,6 +66,7 @@ type clientDealUpdate struct { newState api.DealState id cid.Cid err error + mut func(*ClientDeal) } func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI, chainapi full.ChainAPI) *Client { @@ -137,6 +140,9 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { var deal ClientDeal err := c.deals.Mutate(update.id, func(d *ClientDeal) error { d.State = update.newState + if update.mut != nil { + update.mut(d) + } deal = *d return nil }) diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index a4e87d862..d7264a700 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -1,8 +1,10 @@ package deals import ( + "bytes" "context" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/lib/cborrpc" "golang.org/x/xerrors" @@ -12,11 +14,11 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) -type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error +type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) { go func() { - err := cb(ctx, deal) + mut, err := cb(ctx, deal) if err != nil { next = api.DealError } @@ -30,87 +32,104 @@ func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFu newState: next, id: deal.ProposalCid, err: err, + mut: mut, }: case <-c.stop: } }() } -func (c *Client) new(ctx context.Context, deal ClientDeal) error { +func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { resp, err := c.readStorageDealResp(deal) if err != nil { - return err + return nil, err } if err := c.disconnect(deal); err != nil { - return err + return nil, err } + /* data transfer happens */ if resp.State != api.DealAccepted { - return xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) + return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State) } - // TODO: spec says it's optional - pubmsg, err := c.chain.GetMessage(*resp.PublishMessage) + return func(info *ClientDeal) { + info.PublishMessage = resp.PublishMessage + }, nil +} + +func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { + log.Infow("DEAL ACCEPTED!") + + pubmsg, err := c.chain.GetMessage(*deal.PublishMessage) if err != nil { - return xerrors.Errorf("getting deal pubsish message: %w", err) + return nil, xerrors.Errorf("getting deal pubsish message: %w", err) } pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { - return xerrors.Errorf("getting miner worker failed: %w", err) + return nil, xerrors.Errorf("getting miner worker failed: %w", err) } if pubmsg.From != pw { - return xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) + return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider) } if pubmsg.To != actors.StorageMarketAddress { - return xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) + return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To) } if pubmsg.Method != actors.SMAMethods.PublishStorageDeals { - return xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method) + } + + var params actors.PublishStorageDealsParams + if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil { + return nil, err + } + + dealIdx := -1 + for i, storageDeal := range params.Deals { + // TODO: make it less hacky + eq, err := cborrpc.Equals(&deal.Proposal, &storageDeal.Proposal) + if err != nil { + return nil, err + } + if eq { + dealIdx = i + } + } + + if dealIdx == -1 { + return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *resp.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) if err != nil { - return xerrors.Errorf("waiting for deal publish message: %w", err) + return nil, xerrors.Errorf("waiting for deal publish message: %w", err) } if ret.ExitCode != 0 { - return xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) - } - // TODO: persist dealId - - log.Info("DEAL ACCEPTED!") - - return nil -} - -func (c *Client) accepted(ctx context.Context, deal ClientDeal) error { - /* data transfer happens */ - - resp, err := c.readStorageDealResp(deal) - if err != nil { - return err + return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode) } - if resp.State != api.DealStaged { - return xerrors.Errorf("deal wasn't staged (State=%d)", resp.State) + var res actors.PublishStorageDealResponse + if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + return nil, err } - log.Info("DEAL STAGED!") - - return nil + return func(info *ClientDeal) { + info.DealID = res.DealIDs[dealIdx] + }, nil } -func (c *Client) staged(ctx context.Context, deal ClientDeal) error { - // wait +func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { + // TODO: Maybe wait for pre-commit - return nil + return nil, nil } -func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { +func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) { sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) if err != nil { @@ -172,9 +191,9 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) error { } if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, deal.Proposal.Provider, actors.MAMethods.ProveCommitSector); err != nil { - return xerrors.Errorf("failed to set up called handler") + return nil, xerrors.Errorf("failed to set up called handler") } log.Info("DEAL COMPLETE!!") - return nil + return nil, nil } diff --git a/lib/cborrpc/rpc.go b/lib/cborrpc/rpc.go index 901108457..aee6c5b19 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborrpc/rpc.go @@ -66,3 +66,15 @@ func AsIpld(obj interface{}) (ipld.Node, error) { } return cbor.WrapObject(obj, math.MaxUint64, -1) } + +func Equals(a cbg.CBORMarshaler, b cbg.CBORMarshaler) (bool, error) { + ab, err := Dump(a) + if err != nil { + return false, err + } + bb, err := Dump(b) + if err != nil { + return false, err + } + return bytes.Equal(ab, bb), nil +} From 2588a6a7eb49f1dcd200f7957ff643cf4818fe8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 14:41:41 +0100 Subject: [PATCH 176/230] deals: check msg params in ProveCommitSector event handler --- chain/deals/client_states.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index d7264a700..7ffd3e0ca 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -152,15 +152,41 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal return false, true, nil } - called := func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { - // TODO: handle errors + called := func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) { + defer func() { + if err != nil { + select { + case c.updated <- clientDealUpdate{ + newState: api.DealComplete, + id: deal.ProposalCid, + err: xerrors.Errorf("handling applied event: %w", err), + }: + case <-c.stop: + } + } + }() if msg == nil { log.Error("timed out waiting for deal activation... what now?") return false, nil } - // TODO: can check msg.Params to see if we should even bother checking the state + var params actors.SectorProveCommitInfo + if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil { + return false, err + } + + var found bool + for _, dealID := range params.DealIDs { + if dealID == deal.DealID { + found = true + break + } + } + + if !found { + return true, nil + } sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts) if err != nil { @@ -171,7 +197,7 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal return true, nil } - log.Info("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) + log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) select { case c.updated <- clientDealUpdate{ From 58472afa3af1f603e3d6d981a526e67433895582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 15:09:11 +0100 Subject: [PATCH 177/230] deals: Verify network message signatures --- chain/deals/cbor_gen.go | 36 ++++++++++++++++++++++++++++++---- chain/deals/client.go | 3 +++ chain/deals/client_states.go | 1 - chain/deals/client_utils.go | 4 +++- chain/deals/provider_states.go | 24 +---------------------- chain/deals/provider_utils.go | 3 --- chain/deals/types.go | 10 ++++++++++ node/impl/client/client.go | 20 +++++-------------- 8 files changed, 54 insertions(+), 47 deletions(-) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index c3be140ac..f5bb90d1a 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -404,7 +404,7 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{135}); err != nil { + if _, err := w.Write([]byte{136}); err != nil { return err } @@ -439,6 +439,11 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { return err } + // t.t.MinerWorker (address.Address) (struct) + if err := t.MinerWorker.MarshalCBOR(w); err != nil { + return err + } + // t.t.MinerID (peer.ID) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil { return err @@ -460,7 +465,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 7 { + if extra != 8 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -522,6 +527,15 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { return err } + } + // t.t.MinerWorker (address.Address) (struct) + + { + + if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { + return err + } + } // t.t.MinerID (peer.ID) (string) @@ -541,7 +555,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{134}); err != nil { + if _, err := w.Write([]byte{135}); err != nil { return err } @@ -569,6 +583,11 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } + // t.t.MinerWorker (address.Address) (struct) + if err := t.MinerWorker.MarshalCBOR(w); err != nil { + return err + } + // t.t.DealID (uint64) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err @@ -600,7 +619,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 6 { + if extra != 7 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -645,6 +664,15 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { t.Miner = peer.ID(sval) } + // t.t.MinerWorker (address.Address) (struct) + + { + + if err := t.MinerWorker.UnmarshalCBOR(br); err != nil { + return err + } + + } // t.t.DealID (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) diff --git a/chain/deals/client.go b/chain/deals/client.go index a07c90827..9039737a0 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -35,6 +35,7 @@ type ClientDeal struct { Proposal actors.StorageDealProposal State api.DealState Miner peer.ID + MinerWorker address.Address DealID uint64 PublishMessage *cid.Cid @@ -178,6 +179,7 @@ type ClientDealProposal struct { ProviderAddress address.Address Client address.Address + MinerWorker address.Address MinerID peer.ID } @@ -256,6 +258,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro Proposal: *dealProposal, State: api.DealUnknown, Miner: p.MinerID, + MinerWorker: p.MinerWorker, s: s, } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 7ffd3e0ca..0dce44e3a 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -220,6 +220,5 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal return nil, xerrors.Errorf("failed to set up called handler") } - log.Info("DEAL COMPLETE!!") return nil, nil } diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 7f77dbe0a..d490d401b 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -77,7 +77,9 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { return nil, err } - // TODO: verify signature + if err := resp.Verify(deal.MinerWorker); err != nil { + return nil, xerrors.Errorf("verifying response signature failed", err) + } if resp.Response.Proposal != deal.ProposalCid { return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 468d45f4a..7d94d93e2 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -85,7 +85,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) // TODO: check StorageCollateral - // TODO: minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30)) if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) { return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice) @@ -239,33 +238,12 @@ func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { return nil, xerrors.Errorf("sealing sector failed: %w", err) } - // TODO: Let's not care after this point, for now at least, client can watch the chain - - /*_, err = p.waitSealed(ctx, deal) - if err != nil { - return nil, err - }*/ return nil, nil } func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - /*mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID) - if err != nil { - log.Warnf("Waiting for sector commitment message: %s", err) - }*/ - - //panic("fixme") - - /*err = p.sendSignedResponse(&Response{ - State: api.DealComplete, - Proposal: deal.ProposalCid, - - CommitMessage: &mcid, - }) - if err != nil { - log.Warnf("Sending deal response failed: %s", err) - }*/ + // TODO: observe sector lifecycle, status, expiration.. return nil, nil } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index 0a881d111..aaf8e2c49 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -54,9 +54,6 @@ func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err) } - // TODO: Validate proposal maybe - // (and signature, obviously) - if proposal.DealProposal.Provider != p.actor { log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider) return proposal, err diff --git a/chain/deals/types.go b/chain/deals/types.go index 4318aad39..98b666f5c 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -5,6 +5,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborrpc" "github.com/ipfs/go-cid" ) @@ -36,6 +37,15 @@ type SignedResponse struct { Signature *types.Signature } +func (r *SignedResponse) Verify(addr address.Address) error { + b, err := cborrpc.Dump(&r.Response) + if err != nil { + return err + } + + return r.Signature.Verify(addr, b) +} + type AskRequest struct { Miner address.Address } diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 8856452f1..8f31df6cd 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -24,7 +24,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/store" @@ -61,24 +60,14 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A return nil, xerrors.Errorf("failed to get default address: %w", err) } - // get miner peerID - msg := &types.Message{ - To: miner, - From: miner, - Method: actors.MAMethods.GetPeerID, - } - - r, err := a.StateCall(ctx, msg, nil) + pid, err := a.StateMinerPeerID(ctx, miner, nil) if err != nil { return nil, xerrors.Errorf("failed getting peer ID: %w", err) } - if r.ExitCode != 0 { - return nil, xerrors.Errorf("call to get peer ID for miner failed: exit code %d", r.ExitCode) - } - pid, err := peer.IDFromBytes(r.Return) + mw, err := a.StateMinerWorker(ctx, miner, nil) if err != nil { - return nil, xerrors.Errorf("parsing peer ID wrong: %w", err) + return nil, xerrors.Errorf("failed getting miner worker: %w", err) } proposal := deals.ClientDealProposal{ @@ -86,8 +75,9 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A PricePerEpoch: epochPrice, ProposalExpiration: math.MaxUint64, // TODO: set something reasonable Duration: blocksDuration, - ProviderAddress: miner, Client: self, + ProviderAddress: miner, + MinerWorker: mw, MinerID: pid, } From 3fbe0abb75db37c099895023884d4e8f9d399b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 15:11:39 +0100 Subject: [PATCH 178/230] cborrpc -> cborutil --- chain/blocksync.go | 10 +++++----- chain/deals/client.go | 10 +++++----- chain/deals/client_states.go | 4 ++-- chain/deals/client_utils.go | 4 ++-- chain/deals/provider.go | 4 ++-- chain/deals/provider_asks.go | 14 +++++++------- chain/deals/provider_utils.go | 8 ++++---- chain/deals/types.go | 4 ++-- chain/types/voucher.go | 2 +- lib/{cborrpc => cborutil}/rpc.go | 2 +- lib/statestore/store.go | 10 +++++----- lib/statestore/store_test.go | 4 ++-- node/hello/hello.go | 6 +++--- paych/store.go | 2 +- retrieval/client.go | 10 +++++----- retrieval/miner.go | 14 +++++++------- storage/sector/store.go | 12 ++++++------ storage/sectorblocks/blocks.go | 10 +++++----- 18 files changed, 65 insertions(+), 65 deletions(-) rename lib/{cborrpc => cborutil}/rpc.go (98%) diff --git a/chain/blocksync.go b/chain/blocksync.go index 8a9e629eb..71140f2e0 100644 --- a/chain/blocksync.go +++ b/chain/blocksync.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/node/modules/dtypes" blocks "github.com/ipfs/go-block-format" @@ -93,7 +93,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { defer s.Close() var req BlockSyncRequest - if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &req); err != nil { + if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil { log.Errorf("failed to read block sync request: %s", err) return } @@ -105,7 +105,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) { return } - if err := cborrpc.WriteCborRPC(s, resp); err != nil { + if err := cborutil.WriteCborRPC(s, resp); err != nil { log.Error("failed to write back response for handle stream: ", err) return } @@ -401,12 +401,12 @@ func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *Bloc return nil, err } - if err := cborrpc.WriteCborRPC(s, req); err != nil { + if err := cborutil.WriteCborRPC(s, req); err != nil { return nil, err } var res BlockSyncResponse - if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &res); err != nil { + if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil { return nil, err } diff --git a/chain/deals/client.go b/chain/deals/client.go index 9039737a0..f1101b1e3 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -233,7 +233,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err) } - proposalNd, err := cborrpc.AsIpld(dealProposal) + proposalNd, err := cborutil.AsIpld(dealProposal) if err != nil { return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err) } @@ -248,7 +248,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro Piece: p.Data, } - if err := cborrpc.WriteCborRPC(s, proposal); err != nil { + if err := cborutil.WriteCborRPC(s, proposal); err != nil { s.Reset() return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err) } @@ -280,12 +280,12 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t req := &AskRequest{ Miner: a, } - if err := cborrpc.WriteCborRPC(s, req); err != nil { + if err := cborutil.WriteCborRPC(s, req); err != nil { return nil, xerrors.Errorf("failed to send ask request: %w", err) } var out AskResponse - if err := cborrpc.ReadCborRPC(s, &out); err != nil { + if err := cborutil.ReadCborRPC(s, &out); err != nil { return nil, xerrors.Errorf("failed to read ask response: %w", err) } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 0dce44e3a..5b4b7c78c 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "golang.org/x/xerrors" @@ -91,7 +91,7 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea dealIdx := -1 for i, storageDeal := range params.Deals { // TODO: make it less hacky - eq, err := cborrpc.Equals(&deal.Proposal, &storageDeal.Proposal) + eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal) if err != nil { return nil, err } diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index d490d401b..e4c11eb58 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -9,7 +9,7 @@ import ( unixfile "github.com/ipfs/go-unixfs/file" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" ) @@ -72,7 +72,7 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) { } var resp SignedResponse - if err := cborrpc.ReadCborRPC(s, &resp); err != nil { + if err := cborutil.ReadCborRPC(s, &resp); err != nil { log.Errorw("failed to read Response message", "error", err) return nil, err } diff --git a/chain/deals/provider.go b/chain/deals/provider.go index f54b32cf5..8430cf2f4 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" @@ -193,7 +193,7 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { } func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) { - proposalNd, err := cborrpc.AsIpld(proposal.DealProposal) + proposalNd, err := cborutil.AsIpld(proposal.DealProposal) if err != nil { return MinerDeal{}, err } diff --git a/chain/deals/provider_asks.go b/chain/deals/provider_asks.go index 1b1960b97..1fc8e69ed 100644 --- a/chain/deals/provider_asks.go +++ b/chain/deals/provider_asks.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" datastore "github.com/ipfs/go-datastore" inet "github.com/libp2p/go-libp2p-core/network" "golang.org/x/xerrors" @@ -54,14 +54,14 @@ func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk { func (p *Provider) HandleAskStream(s inet.Stream) { defer s.Close() var ar AskRequest - if err := cborrpc.ReadCborRPC(s, &ar); err != nil { + if err := cborutil.ReadCborRPC(s, &ar); err != nil { log.Errorf("failed to read AskRequest from incoming stream: %s", err) return } resp := p.processAskRequest(&ar) - if err := cborrpc.WriteCborRPC(s, resp); err != nil { + if err := cborutil.WriteCborRPC(s, resp); err != nil { log.Errorf("failed to write ask response: %s", err) return } @@ -98,7 +98,7 @@ func (p *Provider) loadAsk() error { } var ssa types.SignedStorageAsk - if err := cborrpc.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil { return err } @@ -107,7 +107,7 @@ func (p *Provider) loadAsk() error { } func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) { - b, err := cborrpc.Dump(a) + b, err := cborutil.Dump(a) if err != nil { return nil, err } @@ -129,7 +129,7 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) } func (p *Provider) saveAsk(a *types.SignedStorageAsk) error { - b, err := cborrpc.Dump(a) + b, err := cborutil.Dump(a) if err != nil { return err } @@ -150,7 +150,7 @@ func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error { return xerrors.Errorf("failed to get worker for miner in ask", err) } - sigb, err := cborrpc.Dump(ask.Ask) + sigb, err := cborutil.Dump(ask.Ask) if err != nil { return xerrors.Errorf("failed to re-serialize ask") } diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index aaf8e2c49..e14502287 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -8,7 +8,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/ipfs/go-cid" inet "github.com/libp2p/go-libp2p-core/network" @@ -45,7 +45,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) { } func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) { - if err := cborrpc.ReadCborRPC(s, &proposal); err != nil { + if err := cborutil.ReadCborRPC(s, &proposal); err != nil { log.Errorw("failed to read proposal message", "error", err) return proposal, err } @@ -68,7 +68,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error { return xerrors.New("couldn't send response: not connected") } - msg, err := cborrpc.Dump(resp) + msg, err := cborutil.Dump(resp) if err != nil { return xerrors.Errorf("serializing response: %w", err) } @@ -88,7 +88,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error { Signature: sig, } - err = cborrpc.WriteCborRPC(s, signedResponse) + err = cborutil.WriteCborRPC(s, signedResponse) if err != nil { // Assume client disconnected s.Close() diff --git a/chain/deals/types.go b/chain/deals/types.go index 98b666f5c..63e2e45ec 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/ipfs/go-cid" ) @@ -38,7 +38,7 @@ type SignedResponse struct { } func (r *SignedResponse) Verify(addr address.Address) error { - b, err := cborrpc.Dump(&r.Response) + b, err := cborutil.Dump(&r.Response) if err != nil { return err } diff --git a/chain/types/voucher.go b/chain/types/voucher.go index 1cb68406c..1a510d07e 100644 --- a/chain/types/voucher.go +++ b/chain/types/voucher.go @@ -5,7 +5,7 @@ import ( "encoding/base64" "github.com/filecoin-project/lotus/chain/address" - cborrpc "github.com/filecoin-project/lotus/lib/cborrpc" + cborrpc "github.com/filecoin-project/lotus/lib/cborutil" cbor "github.com/ipfs/go-ipld-cbor" ) diff --git a/lib/cborrpc/rpc.go b/lib/cborutil/rpc.go similarity index 98% rename from lib/cborrpc/rpc.go rename to lib/cborutil/rpc.go index aee6c5b19..06af9a156 100644 --- a/lib/cborrpc/rpc.go +++ b/lib/cborutil/rpc.go @@ -1,4 +1,4 @@ -package cborrpc +package cborutil import ( "bytes" diff --git a/lib/statestore/store.go b/lib/statestore/store.go index 80df4a078..c32c3ff22 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -10,7 +10,7 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" ) type StateStore struct { @@ -42,7 +42,7 @@ func (st *StateStore) Begin(i interface{}, state interface{}) error { return xerrors.Errorf("Already tracking state for %s", i) } - b, err := cborrpc.Dump(state) + b, err := cborutil.Dump(state) if err != nil { return err } @@ -68,7 +68,7 @@ func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { return func(in []byte) ([]byte, error) { state := reflect.New(rmut.Type().In(0).Elem()) - err := cborrpc.ReadCborRPC(bytes.NewReader(in), state.Interface()) + err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface()) if err != nil { return nil, err } @@ -79,7 +79,7 @@ func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { return nil, err.(error) } - return cborrpc.Dump(state.Interface()) + return cborutil.Dump(state.Interface()) } } @@ -145,7 +145,7 @@ func (st *StateStore) List(out interface{}) error { } elem := reflect.New(outT) - err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) + err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) if err != nil { return err } diff --git a/lib/statestore/store_test.go b/lib/statestore/store_test.go index 0d17ef20d..91724bbf8 100644 --- a/lib/statestore/store_test.go +++ b/lib/statestore/store_test.go @@ -6,13 +6,13 @@ import ( "github.com/ipfs/go-datastore" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" ) func TestList(t *testing.T) { ds := datastore.NewMapDatastore() - e, err := cborrpc.Dump(types.NewInt(7)) + e, err := cborutil.Dump(types.NewInt(7)) if err != nil { t.Fatal(err) } diff --git a/node/hello/hello.go b/node/hello/hello.go index ef37fae38..90af18923 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/peermgr" ) @@ -66,7 +66,7 @@ func (hs *Service) HandleStream(s inet.Stream) { defer s.Close() var hmsg Message - if err := cborrpc.ReadCborRPC(s, &hmsg); err != nil { + if err := cborutil.ReadCborRPC(s, &hmsg); err != nil { log.Infow("failed to read hello message", "error", err) return } @@ -120,7 +120,7 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error { fmt.Println("SENDING HELLO MESSAGE: ", hts.Cids(), hts.Height()) fmt.Println("hello message genesis: ", gen.Cid()) - if err := cborrpc.WriteCborRPC(s, hmsg); err != nil { + if err := cborutil.WriteCborRPC(s, hmsg); err != nil { return err } diff --git a/paych/store.go b/paych/store.go index 105b87103..f08969630 100644 --- a/paych/store.go +++ b/paych/store.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - cborrpc "github.com/filecoin-project/lotus/lib/cborrpc" + cborrpc "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/retrieval/client.go b/retrieval/client.go index 23e7daa09..8936fbafb 100644 --- a/retrieval/client.go +++ b/retrieval/client.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/retrieval/discovery" @@ -44,7 +44,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid. } defer s.Close() - err = cborrpc.WriteCborRPC(s, &Query{ + err = cborutil.WriteCborRPC(s, &Query{ Piece: data, }) if err != nil { @@ -172,12 +172,12 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out }, } - if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil { + if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil { return err } var resp DealResponse - if err := cborrpc.ReadCborRPC(cst.peeker, &resp); err != nil { + if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil { log.Error(err) return err } @@ -209,7 +209,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error { log.Infof("block %d of %d", i+1, blocksToFetch) var block Block - if err := cborrpc.ReadCborRPC(cst.peeker, &block); err != nil { + if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil { return xerrors.Errorf("reading fetchBlock response: %w", err) } diff --git a/retrieval/miner.go b/retrieval/miner.go index 4a7bd967e..1792880de 100644 --- a/retrieval/miner.go +++ b/retrieval/miner.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -42,7 +42,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner { func writeErr(stream network.Stream, err error) { log.Errorf("Retrieval deal error: %s", err) - _ = cborrpc.WriteCborRPC(stream, &DealResponse{ + _ = cborutil.WriteCborRPC(stream, &DealResponse{ Status: Error, Message: err.Error(), }) @@ -52,7 +52,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { defer stream.Close() var query Query - if err := cborrpc.ReadCborRPC(stream, &query); err != nil { + if err := cborutil.ReadCborRPC(stream, &query); err != nil { writeErr(stream, err) return } @@ -74,7 +74,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) { answer.Size = uint64(size) // TODO: verify on intermediate } - if err := cborrpc.WriteCborRPC(stream, answer); err != nil { + if err := cborutil.WriteCborRPC(stream, answer); err != nil { log.Errorf("Retrieval query: WriteCborRPC: %s", err) return } @@ -114,7 +114,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) { func (hnd *handlerDeal) handleNext() (bool, error) { var deal DealProposal - if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil { + if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil { if err == io.EOF { // client sent all deals err = nil } @@ -203,7 +203,7 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { resp := &DealResponse{ Status: Accepted, } - if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil { + if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil { log.Errorf("Retrieval query: Write Accepted resp: %s", err) return err } @@ -231,7 +231,7 @@ func (hnd *handlerDeal) accept(deal DealProposal) error { Data: nd.RawData(), } - if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil { + if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil { return err } diff --git a/storage/sector/store.go b/storage/sector/store.go index 7f1159090..47bd5af13 100644 --- a/storage/sector/store.go +++ b/storage/sector/store.go @@ -14,7 +14,7 @@ import ( "sync" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -71,7 +71,7 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 var deals DealMapping switch err { case nil: - if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { return 0, err } if deals.Committed { @@ -82,7 +82,7 @@ func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64 deals.DealIDs = append(deals.DealIDs, dealIDs...) deals.Allocated += size - d, err := cborrpc.Dump(&deals) + d, err := cborutil.Dump(&deals) if err != nil { return 0, err } @@ -106,7 +106,7 @@ func (s *Store) PieceSizesToFill(sectorID uint64) ([]uint64, error) { return nil, err } var info DealMapping - if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &info); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(e), &info); err != nil { return nil, err } if info.Allocated > s.sb.SectorSize() { @@ -139,7 +139,7 @@ func (s *Store) DealsForCommit(sectorID uint64, commit bool) ([]uint64, error) { switch err { case nil: var deals DealMapping - if err := cborrpc.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { return nil, err } if !commit { @@ -151,7 +151,7 @@ func (s *Store) DealsForCommit(sectorID uint64, commit bool) ([]uint64, error) { } deals.Committed = true - d, err := cborrpc.Dump(&deals) + d, err := cborutil.Dump(&deals) if err != nil { return nil, err } diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 1d1289def..9f40742d3 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -19,7 +19,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/cborrpc" + "github.com/filecoin-project/lotus/lib/cborutil" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -98,7 +98,7 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si var refs api.SealedRefs if len(v) > 0 { - if err := cborrpc.ReadCborRPC(bytes.NewReader(v), &refs); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(v), &refs); err != nil { return xerrors.Errorf("decoding existing refs: %w", err) } } @@ -109,7 +109,7 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si Size: size, }) - newRef, err := cborrpc.Dump(&refs) + newRef, err := cborutil.Dump(&refs) if err != nil { return xerrors.Errorf("serializing refs: %w", err) } @@ -197,7 +197,7 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { } var refs api.SealedRefs - if err := cborrpc.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil { return nil, err } @@ -217,7 +217,7 @@ func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: t } var refs api.SealedRefs - if err := cborrpc.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil { + if err := cborutil.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil { return nil, err } From 15492697734f634d3c9f39bdd20bb716ba19d5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 15:45:53 +0100 Subject: [PATCH 179/230] Make multiple deals per almost work --- lib/statestore/store.go | 8 ++++++-- storage/sealing.go | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/statestore/store.go b/lib/statestore/store.go index c32c3ff22..994938223 100644 --- a/lib/statestore/store.go +++ b/lib/statestore/store.go @@ -39,7 +39,7 @@ func (st *StateStore) Begin(i interface{}, state interface{}) error { return err } if has { - return xerrors.Errorf("Already tracking state for %s", i) + return xerrors.Errorf("already tracking state for %v", i) } b, err := cborutil.Dump(state) @@ -111,12 +111,16 @@ func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error) return st.ds.Put(k, mutated) } +func (st *StateStore) Has(i interface{}) (bool, error) { + return st.ds.Has(toKey(i)) +} + func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error { k := toKey(i) val, err := st.ds.Get(k) if err != nil { if xerrors.Is(err, datastore.ErrNotFound) { - return xerrors.Errorf("No state for %s", i) + return xerrors.Errorf("No state for %s: %w", i, err) } return err } diff --git a/storage/sealing.go b/storage/sealing.go index 637eb8da2..a2723680d 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -61,6 +61,15 @@ func (m *Miner) sectorStateLoop(ctx context.Context) { } func (m *Miner) onSectorIncoming(sector *SectorInfo) { + has, err := m.sectors.Has(sector.SectorID) + if err != nil { + return + } + if has { + log.Warnf("SealSector called more than once for sector %d", sector.SectorID) + return + } + if err := m.sectors.Begin(sector.SectorID, sector); err != nil { // We may have re-sent the proposal log.Errorf("deal tracking failed: %s", err) From e5f90371c7e27b068af2285134e7d1976b3e7242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 16:19:07 +0100 Subject: [PATCH 180/230] update go-sectorbuilder --- extern/go-sectorbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 2886db51e..181742fdd 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 2886db51e1b49d697d7ee87ea662e4545225acea +Subproject commit 181742fdd08f74f1c9de54fca0f5658706039bc0 From 3e39d6e4453b8a4a7d5b8b14d150e1bafa4c0ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 17:39:27 +0100 Subject: [PATCH 181/230] sectorbuilder: use standalone methods --- lib/sectorbuilder/files.go | 88 ++++++++++++ lib/sectorbuilder/mock.go | 9 +- lib/sectorbuilder/sectorbuilder.go | 173 ++++++++++++++++-------- lib/sectorbuilder/sectorbuilder_test.go | 35 +++-- 4 files changed, 230 insertions(+), 75 deletions(-) create mode 100644 lib/sectorbuilder/files.go diff --git a/lib/sectorbuilder/files.go b/lib/sectorbuilder/files.go new file mode 100644 index 000000000..c0af2b6a7 --- /dev/null +++ b/lib/sectorbuilder/files.go @@ -0,0 +1,88 @@ +package sectorbuilder + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "golang.org/x/xerrors" +) + +func (sb *SectorBuilder) sectorName(sectorID uint64) string { + return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID) +} + +func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string { + return filepath.Join(sb.stagedDir, sb.sectorName(sectorID)) +} + +func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) { + return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) +} + +func (sb *SectorBuilder) sealedSectorPath(sectorID uint64) (string, error) { + path := filepath.Join(sb.sealedDir, sb.sectorName(sectorID)) + + e, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return "", err + } + + return path, e.Close() +} + +func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) { + dir := filepath.Join(sb.cacheDir, sb.sectorName(sectorID)) + + err := os.Mkdir(dir, 0755) + if os.IsExist(err) { + err = nil + } + + return dir, err +} + +func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { + f, ok := r.(*os.File) + if ok { + return f, func() error { return nil }, nil + } + + var w *os.File + + f, w, err := os.Pipe() + if err != nil { + return nil, nil, err + } + + var wait sync.Mutex + var werr error + + wait.Lock() + go func() { + defer wait.Unlock() + + copied, werr := io.CopyN(w, r, n) + if werr != nil { + log.Warnf("toReadableFile: copy error: %+v", werr) + } + + err := w.Close() + if werr == nil && err != nil { + werr = err + log.Warnf("toReadableFile: close error: %+v", err) + return + } + if copied != n { + log.Warnf("copied different amount than expected: %d != %d", copied, n) + werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) + } + }() + + return f, func() error { + wait.Lock() + return werr + }, nil +} diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 451757b17..1433d6a2d 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -24,8 +24,15 @@ func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { staging := filepath.Join(dir, "staging") cache := filepath.Join(dir, "cache") + for _, dir := range []string{metadata, sealed, staging, cache} { + if err := os.Mkdir(dir, 0755); err != nil { + return nil, nil, err + } + } + sb, err := New(&Config{ - SectorSize: sectorSize, + SectorSize: sectorSize, + SealedDir: sealed, StagedDir: staging, MetadataDir: metadata, diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 7f09fa81b..9944429ec 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -2,9 +2,7 @@ package sectorbuilder import ( "io" - "os" "sort" - "sync" "unsafe" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" @@ -15,6 +13,7 @@ import ( ) const PoStReservedWorkers = 1 +const PoRepProofPartitions = 2 var log = logging.Logger("sectorbuilder") @@ -36,6 +35,8 @@ type SealCommitOutput = sectorbuilder.SealCommitOutput type PublicPieceInfo = sectorbuilder.PublicPieceInfo +type RawSealPreCommitOutput = sectorbuilder.RawSealPreCommitOutput + const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { @@ -44,6 +45,10 @@ type SectorBuilder struct { Miner address.Address + stagedDir string + sealedDir string + cacheDir string + rateLimit chan struct{} } @@ -66,7 +71,7 @@ func New(cfg *Config) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) if err != nil { return nil, err } @@ -75,6 +80,10 @@ func New(cfg *Config) (*SectorBuilder, error) { handle: sbp, ssize: cfg.SectorSize, + stagedDir: cfg.StagedDir, + sealedDir: cfg.SealedDir, + cacheDir: cfg.CacheDir, + Miner: cfg.Miner, rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), }, nil @@ -101,21 +110,44 @@ func (sb *SectorBuilder) Destroy() { sectorbuilder.DestroySectorBuilder(sb.handle) } -func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Reader) (uint64, error) { +func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { + return sectorbuilder.AcquireSectorId(sb.handle) +} + +func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader) (PublicPieceInfo, error) { f, werr, err := toReadableFile(file, int64(pieceSize)) if err != nil { - return 0, err + return PublicPieceInfo{}, err } ret := sb.rlimit() defer ret() - sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f) + stagedFile, err := sb.stagedSectorFile(sectorId) if err != nil { - return 0, err + return PublicPieceInfo{}, err } - return sectorID, werr() + writeUnpadded, commP, err := sectorbuilder.StandaloneWriteWithoutAlignment(f, pieceSize, stagedFile) + if err != nil { + return PublicPieceInfo{}, err + } + if writeUnpadded != pieceSize { + return PublicPieceInfo{}, xerrors.Errorf("writeUnpadded != pieceSize: %d != %d", writeUnpadded, pieceSize) + } + + if err := stagedFile.Close(); err != nil { + return PublicPieceInfo{}, err + } + + if err := f.Close(); err != nil { + return PublicPieceInfo{}, err + } + + return PublicPieceInfo{ + Size: pieceSize, + CommP: commP, + }, werr() } // TODO: should *really really* return an io.ReadCloser @@ -126,25 +158,89 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, err return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey) } -func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket) (SealPreCommitOutput, error) { +func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) { ret := sb.rlimit() defer ret() - return sectorbuilder.SealPreCommit(sb.handle, sectorID, ticket) + cacheDir, err := sb.sectorCacheDir(sectorID) + if err != nil { + return RawSealPreCommitOutput{}, err + } + + sealedPath, err := sb.sealedSectorPath(sectorID) + if err != nil { + return RawSealPreCommitOutput{}, err + } + + return sectorbuilder.StandaloneSealPreCommit( + sb.ssize, + PoRepProofPartitions, + cacheDir, + sb.stagedSectorPath(sectorID), + sealedPath, + sectorID, + addressToProverID(sb.Miner), + ticket.TicketBytes, + pieces, + ) } -func (sb *SectorBuilder) SealCommit(sectorID uint64, seed SealSeed) (SealCommitOutput, error) { +func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, pieceKeys []string, rspco RawSealPreCommitOutput) (proof []byte, err error) { ret := sb.rlimit() defer ret() - return sectorbuilder.SealCommit(sb.handle, sectorID, seed) -} + cacheDir, err := sb.sectorCacheDir(sectorID) + if err != nil { + return nil, err + } -func (sb *SectorBuilder) ResumeSealCommit(sectorID uint64) (SealCommitOutput, error) { - ret := sb.rlimit() - defer ret() + proof, err = sectorbuilder.StandaloneSealCommit( + sb.ssize, + PoRepProofPartitions, + cacheDir, + sectorID, + addressToProverID(sb.Miner), + ticket.TicketBytes, + seed.TicketBytes, + pieces, + rspco, + ) + if err != nil { + return nil, xerrors.Errorf("StandaloneSealCommit: %w", err) + } - return sectorbuilder.ResumeSealCommit(sb.handle, sectorID) + pmeta := make([]sectorbuilder.PieceMetadata, len(pieces)) + for i, piece := range pieces { + pmeta[i] = sectorbuilder.PieceMetadata{ + Key: pieceKeys[i], + Size: piece.Size, + CommP: piece.CommP, + } + } + + sealedPath, err := sb.sealedSectorPath(sectorID) + if err != nil { + return nil, err + } + + err = sectorbuilder.ImportSealedSector( + sb.handle, + sectorID, + cacheDir, + sealedPath, + ticket, + seed, + rspco.CommR, + rspco.CommD, + rspco.CommC, + rspco.CommRLast, + proof, + pmeta, + ) + if err != nil { + return nil, xerrors.Errorf("ImportSealedSector: %w", err) + } + return proof, nil } func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { @@ -217,46 +313,3 @@ func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen] func GenerateDataCommitment(ssize uint64, pieces []PublicPieceInfo) ([CommLen]byte, error) { return sectorbuilder.GenerateDataCommitment(ssize, pieces) } - -func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { - f, ok := r.(*os.File) - if ok { - return f, func() error { return nil }, nil - } - - var w *os.File - - f, w, err := os.Pipe() - if err != nil { - return nil, nil, err - } - - var wait sync.Mutex - var werr error - - wait.Lock() - go func() { - defer wait.Unlock() - - copied, werr := io.CopyN(w, r, n) - if werr != nil { - log.Warnf("toReadableFile: copy error: %+v", werr) - } - - err := w.Close() - if werr == nil && err != nil { - werr = err - log.Warnf("toReadableFile: close error: %+v", err) - return - } - if copied != n { - log.Warnf("copied different amount than expected: %d != %d", copied, n) - werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) - } - }() - - return f, func() error { - wait.Lock() - return werr - }, nil -} diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 5ce934f04..c83f2c97b 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -3,6 +3,7 @@ package sectorbuilder_test import ( "io" "math/rand" + "os" "testing" "github.com/filecoin-project/lotus/build" @@ -12,27 +13,33 @@ import ( const sectorSize = 1024 func TestSealAndVerify(t *testing.T) { - t.Skip("this is slow") - //os.Setenv("BELLMAN_NO_GPU", "1") + //t.Skip("this is slow") + os.Setenv("BELLMAN_NO_GPU", "1") build.SectorSizes = []uint64{sectorSize} if err := build.GetParams(true); err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } - defer cleanup() + _ = cleanup + //defer cleanup() dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) - r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) - sid, err := sb.AddPiece("foo", dlen, r) + sid, err := sb.AcquireSectorId() if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) + } + + r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) + ppi, err := sb.AddPiece(dlen, sid, r) + if err != nil { + t.Fatalf("%+v", err) } ticket := sectorbuilder.SealTicket{ @@ -40,9 +47,9 @@ func TestSealAndVerify(t *testing.T) { TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, } - pco, err := sb.SealPreCommit(sid, ticket) + pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{ppi}) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } seed := sectorbuilder.SealSeed{ @@ -50,14 +57,14 @@ func TestSealAndVerify(t *testing.T) { TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}, } - sco, err := sb.SealCommit(sid, seed) + proof, err := sb.SealCommit(sid, ticket, seed, []sectorbuilder.PublicPieceInfo{ppi}, []string{"foo"}, pco) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } - ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], pco.CommD[:], sb.Miner, ticket.TicketBytes[:], seed.TicketBytes[:], sid, sco.Proof) + ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], pco.CommD[:], sb.Miner, ticket.TicketBytes[:], seed.TicketBytes[:], sid, proof) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } if !ok { From 69b4bd9fb40376e2f7edf2490d95c4aab42ed5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 17:43:59 +0100 Subject: [PATCH 182/230] sectorbuilder: Also test PoSt --- lib/sectorbuilder/sectorbuilder_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index c83f2c97b..1b4933656 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -70,4 +70,24 @@ func TestSealAndVerify(t *testing.T) { if !ok { t.Fatal("proof failed to validate") } + + cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9} + + ssi := sectorbuilder.NewSortedSectorInfo([]sectorbuilder.SectorInfo{{ + SectorID: sid, + CommR: pco.CommR, + }}) + + postProof, err := sb.GeneratePoSt(ssi, cSeed, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + + ok, err = sectorbuilder.VerifyPost(sb.SectorSize(), ssi, cSeed, postProof, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + if !ok { + t.Fatal("bad post") + } } From 2d26a4edf7a488376065245d929bfde163e01cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 19:22:59 +0100 Subject: [PATCH 183/230] Sector storage refactor --- chain/deals/provider_states.go | 8 +- gen/main.go | 10 +- node/builder.go | 4 +- node/impl/storminer.go | 2 - node/modules/storageminer.go | 7 +- storage/garbage.go | 34 +-- storage/miner.go | 15 +- storage/post.go | 22 +- storage/sealing.go | 104 +++++++- storage/sealing_utils.go | 20 ++ .../store_test.go => sealing_utils_test.go} | 41 +-- storage/sector/cbor_gen.go | 119 --------- storage/sector/store.go | 238 ------------------ storage/sector_states.go | 62 +++-- storage/sectorblocks/blocks.go | 12 +- 15 files changed, 213 insertions(+), 485 deletions(-) create mode 100644 storage/sealing_utils.go rename storage/{sector/store_test.go => sealing_utils_test.go} (55%) delete mode 100644 storage/sector/cbor_gen.go delete mode 100644 storage/sector/store.go diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 7d94d93e2..00546202d 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -219,7 +219,7 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size") } - sectorID, err := p.secb.AddUnixfsPiece(deal.Ref, uf, deal.DealID) + sectorID, err := p.secb.AddUnixfsPiece(ctx, deal.Ref, uf, deal.DealID) if err != nil { return nil, xerrors.Errorf("AddPiece failed: %s", err) } @@ -228,16 +228,12 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return func(deal *MinerDeal) { deal.SectorID = sectorID }, nil - } // SEALING func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - log.Info("About to seal sector!", deal.ProposalCid, deal.SectorID) - if err := p.sminer.SealSector(ctx, deal.SectorID); err != nil { - return nil, xerrors.Errorf("sealing sector failed: %w", err) - } + // TODO: consider waiting for seal to happen return nil, nil } diff --git a/gen/main.go b/gen/main.go index efe1af965..2c9292283 100644 --- a/gen/main.go +++ b/gen/main.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sector" ) func main() { @@ -153,18 +152,11 @@ func main() { err = gen.WriteTupleEncodersToFile("./storage/cbor_gen.go", "storage", storage.SealTicket{}, + storage.Piece{}, storage.SectorInfo{}, ) if err != nil { fmt.Println(err) os.Exit(1) } - - err = gen.WriteTupleEncodersToFile("./storage/sector/cbor_gen.go", "sector", - sector.DealMapping{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } } diff --git a/node/builder.go b/node/builder.go index 145a92fe1..cf999ea4b 100644 --- a/node/builder.go +++ b/node/builder.go @@ -41,7 +41,6 @@ import ( "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/retrieval/discovery" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -229,9 +228,8 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), - Override(new(*sector.Store), sector.NewStore), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), - Override(new(sector.TicketFn), modules.SealTicketGen), + Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), Override(new(dtypes.StagingDAG), modules.StagingDAG), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b9efe9f2f..e7b081ea6 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -7,7 +7,6 @@ import ( "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sector" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -16,7 +15,6 @@ type StorageMinerAPI struct { SectorBuilderConfig *sectorbuilder.Config SectorBuilder *sectorbuilder.SectorBuilder - Sectors *sector.Store SectorBlocks *sectorblocks.SectorBlocks Miner *storage.Miner diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 5a9ea4d75..3c827c550 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -28,7 +28,6 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/retrieval" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sector" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -81,13 +80,13 @@ func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataD } } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, secst *sector.Store) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err } - sm, err := storage.NewMiner(api, maddr, h, ds, secst) + sm, err := storage.NewMiner(api, maddr, h, ds, sb, tktFn) if err != nil { return nil, err } @@ -177,7 +176,7 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro return nil } -func SealTicketGen(api api.FullNode) sector.TicketFn { +func SealTicketGen(api api.FullNode) storage.TicketFn { return func(ctx context.Context) (*sectorbuilder.SealTicket, error) { ts, err := api.ChainHead(ctx) if err != nil { diff --git a/storage/garbage.go b/storage/garbage.go index 8e3cf1a10..8a2dd7a22 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -17,7 +17,7 @@ import ( ) // TODO: expected sector ID -func (m *Miner) storeGarbage(ctx context.Context, sizes ...uint64) ([]uint64, error) { +func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, sizes ...uint64) ([]Piece, error) { deals := make([]actors.StorageDeal, len(sizes)) for i, size := range sizes { commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) @@ -87,41 +87,47 @@ func (m *Miner) storeGarbage(ctx context.Context, sizes ...uint64) ([]uint64, er return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals") } - sectorIDs := make([]uint64, len(sizes)) + out := make([]Piece, len(sizes)) for i, size := range sizes { name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - sectorID, err := m.secst.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), resp.DealIDs[i]) + ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) if err != nil { return nil, err } - sectorIDs[i] = sectorID + out[i] = Piece{ + DealID: resp.DealIDs[i], + Ref: name, + Size: ppi.Size, + CommP: ppi.CommP[:], + } } - return sectorIDs, nil + return out, nil } func (m *Miner) StoreGarbageData(_ context.Context) error { ctx := context.TODO() - ssize, err := m.SectorSize(ctx) - if err != nil { - return xerrors.Errorf("failed to get miner sector size: %w", err) - } go func() { - size := sectorbuilder.UserBytesForSectorSize(ssize) + size := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize()) - sids, err := m.storeGarbage(ctx, size) + sid, err := m.sb.AcquireSectorId() if err != nil { log.Errorf("%+v", err) return } - if err := m.SealSector(context.TODO(), sids[0]); err != nil { + pieces, err := m.storeGarbage(ctx, sid, size) + if err != nil { + log.Errorf("%+v", err) + return + } + + if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].Ref, pieces[0].ppi()); err != nil { log.Errorf("%+v", err) return } }() - - return err + return nil } diff --git a/storage/miner.go b/storage/miner.go index 72a6a0a8b..c3ddcc8e5 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/lib/sectorbuilder" "sync" "github.com/filecoin-project/lotus/lib/statestore" @@ -18,7 +19,6 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sector" ) var log = logging.Logger("storageminer") @@ -29,7 +29,6 @@ type Miner struct { api storageMinerApi events *events.Events h host.Host - secst *sector.Store maddr address.Address worker address.Address @@ -39,7 +38,9 @@ type Miner struct { schedPost uint64 // Sealing + sb *sectorbuilder.SectorBuilder sectors *statestore.StateStore + tktFn TicketFn sectorIncoming chan *SectorInfo sectorUpdated chan sectorUpdate @@ -73,13 +74,14 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, secst *sector.Store) (*Miner, error) { +func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, tktFn TicketFn) (*Miner, error) { return &Miner{ api: api, maddr: addr, h: h, - secst: secst, + sb: sb, + tktFn: tktFn, sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))), @@ -132,8 +134,3 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { log.Infof("starting up miner %s, worker addr %s", m.maddr, m.worker) return nil } - -func (m *Miner) SectorSize(ctx context.Context) (uint64, error) { - // TODO: cache this - return m.api.StateMinerSectorSize(ctx, m.maddr, nil) -} diff --git a/storage/post.go b/storage/post.go index ff21c48f7..0f4c74f38 100644 --- a/storage/post.go +++ b/storage/post.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/lib/sectorbuilder" "time" "github.com/ipfs/go-cid" @@ -158,6 +159,21 @@ func (p *post) preparePost(ctx context.Context) error { return nil } +func (p *post) sortedSectorInfo() sectorbuilder.SortedSectorInfo { + sbsi := make([]sectorbuilder.SectorInfo, len(p.sset)) + for k, sector := range p.sset { + var commR [sectorbuilder.CommLen]byte + copy(commR[:], sector.CommR) + + sbsi[k] = sectorbuilder.SectorInfo{ + SectorID: sector.SectorID, + CommR: commR, + } + } + + return sectorbuilder.NewSortedSectorInfo(sbsi) +} + func (p *post) runPost(ctx context.Context) error { ctx, span := trace.StartSpan(ctx, "storage.runPost") defer span.End() @@ -168,7 +184,11 @@ func (p *post) runPost(ctx context.Context) error { tsStart := time.Now() var faults []uint64 // TODO - proof, err := p.m.secst.RunPoSt(ctx, p.sset, p.r, faults) + + var seed [32]byte + copy(seed[:], p.r) + + proof, err := p.m.sb.GeneratePoSt(p.sortedSectorInfo(), seed, faults) if err != nil { return xerrors.Errorf("running post failed: %w", err) } diff --git a/storage/sealing.go b/storage/sealing.go index a2723680d..4000d46fe 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,25 +2,56 @@ package storage import ( "context" + "io" - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" xerrors "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/types" ) +type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) + type SealTicket struct { BlockHeight uint64 TicketBytes []byte } +func (t *SealTicket) sb() sectorbuilder.SealTicket { + out := sectorbuilder.SealTicket{BlockHeight: t.BlockHeight} + copy(out.TicketBytes[:], t.TicketBytes) + return out +} + +type Piece struct { + DealID uint64 + Ref string + + Size uint64 + CommP []byte +} + +func (p *Piece) ppi() (out sectorbuilder.PublicPieceInfo) { + out.Size = p.Size + copy(out.CommP[:], p.CommP) + return out +} + type SectorInfo struct { State api.SectorState SectorID uint64 + // Packing + + Pieces []Piece + // PreCommit - CommD []byte - CommR []byte + CommC []byte + CommD []byte + CommR []byte + CommRLast []byte Ticket SealTicket PreCommitMessage *cid.Cid @@ -40,6 +71,41 @@ type sectorUpdate struct { mut func(*SectorInfo) } +func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo { + out := make([]sectorbuilder.PublicPieceInfo, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.ppi() + } + return out +} + +func (t *SectorInfo) deals() []uint64 { + out := make([]uint64, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.DealID + } + return out +} + +func (t *SectorInfo) refs() []string { + out := make([]string, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.Ref + } + return out +} + +func (t *SectorInfo) rspco() sectorbuilder.RawSealPreCommitOutput { + var out sectorbuilder.RawSealPreCommitOutput + + copy(out.CommC[:], t.CommC) + copy(out.CommD[:], t.CommD) + copy(out.CommR[:], t.CommR) + copy(out.CommRLast[:], t.CommRLast) + + return out +} + func (m *Miner) sectorStateLoop(ctx context.Context) { // TODO: restore state @@ -66,7 +132,7 @@ func (m *Miner) onSectorIncoming(sector *SectorInfo) { return } if has { - log.Warnf("SealSector called more than once for sector %d", sector.SectorID) + log.Warnf("SealPiece called more than once for sector %d", sector.SectorID) return } @@ -129,12 +195,36 @@ func (m *Miner) failSector(id uint64, err error) { panic(err) // todo: better error handling strategy } -func (m *Miner) SealSector(ctx context.Context, sid uint64) error { - log.Infof("Begin sealing sector %d", sid) +func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Reader, dealID uint64) (uint64, error) { + log.Infof("Seal piece for deal %d", dealID) + sid, err := m.sb.AcquireSectorId() // TODO: Put more than one thing in a sector + if err != nil { + return 0, xerrors.Errorf("acquiring sector ID: %w", err) + } + + ppi, err := m.sb.AddPiece(size, sid, r) + if err != nil { + return 0, xerrors.Errorf("adding piece to sector: %w", err) + } + + return sid, m.newSector(ctx, sid, dealID, ref, ppi) +} + +func (m *Miner) newSector(ctx context.Context, sid uint64, dealID uint64, ref string, ppi sectorbuilder.PublicPieceInfo) error { si := &SectorInfo{ State: api.UndefinedSectorState, SectorID: sid, + + Pieces: []Piece{ + { + DealID: dealID, + Ref:ref, + + Size: ppi.Size, + CommP: ppi.CommP[:], + }, + }, } select { case m.sectorIncoming <- si: diff --git a/storage/sealing_utils.go b/storage/sealing_utils.go new file mode 100644 index 000000000..17d83abbf --- /dev/null +++ b/storage/sealing_utils.go @@ -0,0 +1,20 @@ +package storage + +import ( + "math/bits" + + "github.com/filecoin-project/lotus/lib/sectorbuilder" +) + +func fillersFromRem(toFill uint64) ([]uint64, error) { + toFill += toFill / 127 // convert to in-sector bytes for easier math + + out := make([]uint64, bits.OnesCount64(toFill)) + for i := range out { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << next + toFill ^= psize + out[i] = sectorbuilder.UserBytesForSectorSize(psize) + } + return out, nil +} diff --git a/storage/sector/store_test.go b/storage/sealing_utils_test.go similarity index 55% rename from storage/sector/store_test.go rename to storage/sealing_utils_test.go index 78d7ccd0e..6a975e664 100644 --- a/storage/sector/store_test.go +++ b/storage/sealing_utils_test.go @@ -1,18 +1,11 @@ -package sector +package storage import ( - "context" - "fmt" - "github.com/filecoin-project/lotus/lib/padreader" - "io" - "math/rand" "testing" "github.com/stretchr/testify/assert" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/ipfs/go-datastore" ) func testFill(t *testing.T, n uint64, exp []uint64) { @@ -51,35 +44,3 @@ func TestFillersFromRem(t *testing.T) { } } - -func TestSectorStore(t *testing.T) { - if err := build.GetParams(true); err != nil { - t.Fatal(err) - } - - sb, cleanup, err := sectorbuilder.TempSectorbuilder(1024) - if err != nil { - t.Fatal(err) - } - defer cleanup() - - tktFn := func(context.Context) (*sectorbuilder.SealTicket, error) { - return §orbuilder.SealTicket{ - BlockHeight: 17, - TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2}, - }, nil - } - - ds := datastore.NewMapDatastore() - - store := NewStore(sb, ds, tktFn) - - pr := io.LimitReader(rand.New(rand.NewSource(17)), 300) - pr, n := padreader.New(pr, 300) - - sid, err := store.AddPiece("a", n, pr, 1) - if err != nil { - t.Fatal(err) - } - fmt.Println(sid) -} diff --git a/storage/sector/cbor_gen.go b/storage/sector/cbor_gen.go deleted file mode 100644 index 11a2365f1..000000000 --- a/storage/sector/cbor_gen.go +++ /dev/null @@ -1,119 +0,0 @@ -package sector - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -/* This file was generated by github.com/whyrusleeping/cbor-gen */ - -var _ = xerrors.Errorf - -func (t *DealMapping) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.t.DealIDs ([]uint64) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil { - return err - } - for _, v := range t.DealIDs { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil { - return err - } - } - - // t.t.Allocated (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Allocated))); err != nil { - return err - } - - // t.t.Committed (bool) (bool) - if err := cbg.WriteBool(w, t.Committed); err != nil { - return err - } - return nil -} - -func (t *DealMapping) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.t.DealIDs ([]uint64) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if extra > 8192 { - return fmt.Errorf("t.DealIDs: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.DealIDs = make([]uint64, extra) - } - for i := 0; i < int(extra); i++ { - - maj, val, err := cbg.CborReadHeader(br) - if err != nil { - return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err) - } - - if maj != cbg.MajUnsignedInt { - return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj) - } - - t.DealIDs[i] = val - } - - // t.t.Allocated (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Allocated = uint64(extra) - // t.t.Committed (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Committed = false - case 21: - t.Committed = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - return nil -} diff --git a/storage/sector/store.go b/storage/sector/store.go deleted file mode 100644 index 47bd5af13..000000000 --- a/storage/sector/store.go +++ /dev/null @@ -1,238 +0,0 @@ -package sector - -import ( - "bytes" - "context" - "fmt" - "github.com/filecoin-project/go-sectorbuilder/sealing_state" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - logging "github.com/ipfs/go-log" - "golang.org/x/xerrors" - "io" - "math/bits" - "sync" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/cborutil" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -var log = logging.Logger("sectorstore") - -var sectorDealsPrefix = datastore.NewKey("/sectordeals") - -type DealMapping struct { - DealIDs []uint64 - Allocated uint64 - Committed bool -} - -type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) - -// TODO: eventually handle sector storage here instead of in rust-sectorbuilder -type Store struct { - sb *sectorbuilder.SectorBuilder - tktFn TicketFn - - dealsLk sync.Mutex - deals datastore.Datastore -} - -func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store { - return &Store{ - sb: sb, - tktFn: tktFn, - deals: namespace.Wrap(ds, sectorDealsPrefix), - } -} - -func (s *Store) SectorStatus(sid uint64) (*sectorbuilder.SectorSealingStatus, error) { - status, err := s.sb.SealStatus(sid) - if err != nil { - return nil, err - } - - return &status, nil -} - -func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) { - sectorID, err = s.sb.AddPiece(ref, size, r) - if err != nil { - return 0, err - } - - s.dealsLk.Lock() - defer s.dealsLk.Unlock() - - k := datastore.NewKey(fmt.Sprint(sectorID)) - e, err := s.deals.Get(k) - var deals DealMapping - switch err { - case nil: - if err := cborutil.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { - return 0, err - } - if deals.Committed { - return 0, xerrors.Errorf("sector %d already committed", sectorID) - } - fallthrough - case datastore.ErrNotFound: - deals.DealIDs = append(deals.DealIDs, dealIDs...) - deals.Allocated += size - - d, err := cborutil.Dump(&deals) - if err != nil { - return 0, err - } - if err := s.deals.Put(k, d); err != nil { - return 0, err - } - default: - return 0, err - } - - return sectorID, nil -} - -func (s *Store) PieceSizesToFill(sectorID uint64) ([]uint64, error) { - s.dealsLk.Lock() - defer s.dealsLk.Unlock() - - k := datastore.NewKey(fmt.Sprint(sectorID)) - e, err := s.deals.Get(k) - if err != nil { - return nil, err - } - var info DealMapping - if err := cborutil.ReadCborRPC(bytes.NewReader(e), &info); err != nil { - return nil, err - } - if info.Allocated > s.sb.SectorSize() { - return nil, xerrors.Errorf("more data allocated in sector than should be able to fit: %d > %d", info.Allocated, s.sb.SectorSize()) - } - - return fillersFromRem(sectorbuilder.UserBytesForSectorSize(s.sb.SectorSize()) - info.Allocated) -} - -func fillersFromRem(toFill uint64) ([]uint64, error) { - toFill += toFill / 127 // convert to in-sector bytes for easier math - - out := make([]uint64, bits.OnesCount64(toFill)) - for i := range out { - next := bits.TrailingZeros64(toFill) - psize := uint64(1) << next - toFill ^= psize - out[i] = sectorbuilder.UserBytesForSectorSize(psize) - } - return out, nil -} - -func (s *Store) DealsForCommit(sectorID uint64, commit bool) ([]uint64, error) { - s.dealsLk.Lock() - defer s.dealsLk.Unlock() - - k := datastore.NewKey(fmt.Sprint(sectorID)) - e, err := s.deals.Get(k) - - switch err { - case nil: - var deals DealMapping - if err := cborutil.ReadCborRPC(bytes.NewReader(e), &deals); err != nil { - return nil, err - } - if !commit { - return nil, nil - } - - if deals.Committed { - log.Errorf("getting deal IDs for sector %d: sector already marked as committed", sectorID) - } - - deals.Committed = true - d, err := cborutil.Dump(&deals) - if err != nil { - return nil, err - } - if err := s.deals.Put(k, d); err != nil { - return nil, err - } - - return deals.DealIDs, nil - case datastore.ErrNotFound: - log.Errorf("getting deal IDs for sector %d failed: %s", err) - return []uint64{}, nil - default: - return nil, err - } -} - -func (s *Store) SealPreCommit(ctx context.Context, sectorID uint64) (sectorbuilder.SealPreCommitOutput, error) { - tkt, err := s.tktFn(ctx) - if err != nil { - return sectorbuilder.SealPreCommitOutput{}, err - } - - return s.sb.SealPreCommit(sectorID, *tkt) -} - -func (s *Store) SealComputeProof(ctx context.Context, sectorID uint64, height uint64, rand []byte) ([]byte, error) { - var tick [32]byte - copy(tick[:], rand) - - sco, err := s.sb.SealCommit(sectorID, sectorbuilder.SealSeed{ - BlockHeight: height, - TicketBytes: tick, - }) - if err != nil { - return nil, err - } - return sco.Proof, nil -} - -func (s *Store) Committed() ([]sectorbuilder.SectorSealingStatus, error) { - l, err := s.sb.GetAllStagedSectors() - if err != nil { - return nil, err - } - - out := make([]sectorbuilder.SectorSealingStatus, 0) - for _, sid := range l { - status, err := s.sb.SealStatus(sid) - if err != nil { - return nil, err - } - - if status.State != sealing_state.Committed { - continue - } - out = append(out, status) - } - - return out, nil -} - -func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte, faults []uint64) ([]byte, error) { - sbsi := make([]sectorbuilder.SectorInfo, len(sectors)) - for k, sector := range sectors { - var commR [sectorbuilder.CommLen]byte - if copy(commR[:], sector.CommR) != sectorbuilder.CommLen { - return nil, xerrors.Errorf("commR too short, %d bytes", len(sector.CommR)) - } - - sbsi[k] = sectorbuilder.SectorInfo{ - SectorID: sector.SectorID, - CommR: commR, - } - } - - ssi := sectorbuilder.NewSortedSectorInfo(sbsi) - - var seed [sectorbuilder.CommLen]byte - if copy(seed[:], r) != sectorbuilder.CommLen { - return nil, xerrors.Errorf("random seed too short, %d bytes", len(r)) - } - - return s.sb.GeneratePoSt(ssi, seed, faults) -} diff --git a/storage/sector_states.go b/storage/sector_states.go index b52402c4f..ded491192 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -37,7 +38,16 @@ func (m *Miner) handle(ctx context.Context, sector SectorInfo, cb providerHandle func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { log.Infow("performing filling up rest of the sector...", "sector", sector.SectorID) - fillerSizes, err := m.secst.PieceSizesToFill(sector.SectorID) + var allocated uint64 + for _, piece := range sector.Pieces { + allocated += piece.Size + } + + if allocated > m.sb.SectorSize() { + return nil, xerrors.Errorf("too much data in sector: %d > %d", allocated, m.sb.SectorSize()) + } + + fillerSizes, err := fillersFromRem(m.sb.SectorSize() - allocated) if err != nil { return nil, err } @@ -46,49 +56,47 @@ func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*Sec log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID) } - ids, err := m.storeGarbage(ctx, fillerSizes...) + pieces, err := m.storeGarbage(ctx, sector.SectorID, fillerSizes...) if err != nil { return nil, xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) } - for _, id := range ids { - if id != sector.SectorID { - panic("todo: pass SectorID into storeGarbage") - } - } - - return nil, nil + return func(info *SectorInfo) { + info.Pieces = append(info.Pieces, pieces...) + }, nil } func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { log.Infow("performing sector replication...", "sector", sector.SectorID) - sinfo, err := m.secst.SealPreCommit(ctx, sector.SectorID) + ticket, err := m.tktFn(ctx) + if err != nil { + return nil, err + } + + rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos()) if err != nil { return nil, xerrors.Errorf("seal pre commit failed: %w", err) } return func(info *SectorInfo) { - info.CommD = sinfo.CommD[:] - info.CommR = sinfo.CommR[:] + info.CommC = rspco.CommC[:] + info.CommD = rspco.CommD[:] + info.CommR = rspco.CommR[:] + info.CommRLast = rspco.CommRLast[:] info.Ticket = SealTicket{ - BlockHeight: sinfo.Ticket.BlockHeight, - TicketBytes: sinfo.Ticket.TicketBytes[:], + BlockHeight: ticket.BlockHeight, + TicketBytes: ticket.TicketBytes[:], } }, nil } func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { - deals, err := m.secst.DealsForCommit(sector.SectorID, false) - if err != nil { - return nil, err - } - params := &actors.SectorPreCommitInfo{ SectorNumber: sector.SectorID, CommR: sector.CommR, SealEpoch: sector.Ticket.BlockHeight, - DealIDs: deals, + DealIDs: sector.deals(), } enc, aerr := actors.SerializeParams(params) if aerr != nil { @@ -164,20 +172,20 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector return nil, xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) } - proof, err := m.secst.SealComputeProof(ctx, sector.SectorID, sector.RandHeight, rand) + seed := sectorbuilder.SealSeed{ + BlockHeight: sector.RandHeight, + } + copy(seed.TicketBytes[:], rand) + + proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.sb(), seed, sector.pieceInfos(), sector.refs(), sector.rspco()) if err != nil { return nil, xerrors.Errorf("computing seal proof failed: %w", err) } - deals, err := m.secst.DealsForCommit(sector.SectorID, true) - if err != nil { - return nil, err - } - params := &actors.SectorProveCommitInfo{ Proof: proof, SectorID: sector.SectorID, - DealIDs: deals, + DealIDs: sector.deals(), } enc, aerr := actors.SerializeParams(params) diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index 9f40742d3..e1f64ae3e 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sector" + "github.com/filecoin-project/lotus/storage" ) type SealSerialization uint8 @@ -38,7 +38,7 @@ var imBlocksPrefix = datastore.NewKey("/intermediate") var ErrNotFound = errors.New("not found") type SectorBlocks struct { - *sector.Store + *storage.Miner intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore @@ -47,9 +47,9 @@ type SectorBlocks struct { keyLk sync.Mutex } -func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { +func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks { sbc := &SectorBlocks{ - Store: sectst, + Miner: miner, intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)), @@ -160,7 +160,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) { } } -func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) { +func (st *SectorBlocks) AddUnixfsPiece(ctx context.Context, ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) { size, err := r.Size() if err != nil { return 0, err @@ -175,7 +175,7 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6 pr, psize := padreader.New(r, uint64(size)) - return st.Store.AddPiece(refst.pieceRef, psize, pr, dealID) + return st.Miner.SealPiece(ctx, refst.pieceRef, psize, pr, dealID) } func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) { From 68adf519c7fed001541d160236f5458893f4ebc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 19:27:23 +0100 Subject: [PATCH 184/230] cbor-gen --- storage/cbor_gen.go | 192 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 2 deletions(-) diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 74c5f93e6..c84a32590 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -82,12 +82,114 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *Piece) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{132}); err != nil { + return err + } + + // t.t.DealID (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.t.Ref (string) (string) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Ref)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Ref)); err != nil { + return err + } + + // t.t.Size (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + return err + } + + // t.t.CommP ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil { + return err + } + if _, err := w.Write(t.CommP); err != nil { + return err + } + return nil +} + +func (t *Piece) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.DealID (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.t.Ref (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Ref = string(sval) + } + // t.t.Size (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + // t.t.CommP ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommP: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommP = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommP); err != nil { + return err + } + return nil +} + func (t *SectorInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{137}); err != nil { + if _, err := w.Write([]byte{140}); err != nil { return err } @@ -101,6 +203,24 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + // t.t.Pieces ([]storage.Piece) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { + return err + } + for _, v := range t.Pieces { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.t.CommC ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommC)))); err != nil { + return err + } + if _, err := w.Write(t.CommC); err != nil { + return err + } + // t.t.CommD ([]uint8) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil { return err @@ -117,6 +237,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + // t.t.CommRLast ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommRLast)))); err != nil { + return err + } + if _, err := w.Write(t.CommRLast); err != nil { + return err + } + // t.t.Ticket (storage.SealTicket) (struct) if err := t.Ticket.MarshalCBOR(w); err != nil { return err @@ -170,7 +298,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 9 { + if extra != 12 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -194,6 +322,49 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.SectorID = uint64(extra) + // t.t.Pieces ([]storage.Piece) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Pieces: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Pieces = make([]Piece, extra) + } + for i := 0; i < int(extra); i++ { + + var v Piece + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Pieces[i] = v + } + + // t.t.CommC ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommC: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommC = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommC); err != nil { + return err + } // t.t.CommD ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) @@ -228,6 +399,23 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommR); err != nil { return err } + // t.t.CommRLast ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.CommRLast: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommRLast = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommRLast); err != nil { + return err + } // t.t.Ticket (storage.SealTicket) (struct) { From 58c21b4a5e4c03669af7f99436f60025ccbfde67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 19:33:46 +0100 Subject: [PATCH 185/230] sectorbuilder: Always create directories --- lib/sectorbuilder/mock.go | 6 ------ lib/sectorbuilder/sectorbuilder.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 1433d6a2d..20ec4ac51 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -24,12 +24,6 @@ func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { staging := filepath.Join(dir, "staging") cache := filepath.Join(dir, "cache") - for _, dir := range []string{metadata, sealed, staging, cache} { - if err := os.Mkdir(dir, 0755); err != nil { - return nil, nil, err - } - } - sb, err := New(&Config{ SectorSize: sectorSize, diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 9944429ec..98e0dcfd1 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -2,6 +2,7 @@ package sectorbuilder import ( "io" + "os" "sort" "unsafe" @@ -71,6 +72,15 @@ func New(cfg *Config) (*SectorBuilder, error) { proverId := addressToProverID(cfg.Miner) + for _, dir := range []string{cfg.StagedDir, cfg.SealedDir, cfg.CacheDir, cfg.MetadataDir} { + if err := os.Mkdir(dir, 0755); err != nil { + if os.IsExist(err) { + continue + } + return nil, err + } + } + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) if err != nil { return nil, err From f40eb8a521260fbd4d892905dade3fd195f76da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 19:43:15 +0100 Subject: [PATCH 186/230] More post-refactor fixes --- api/test/deals.go | 5 ++++- storage/miner.go | 6 +++--- storage/sealing.go | 14 +++++++------- storage/sector_states.go | 8 +++++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 3145f7c4a..8e8f37388 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -12,6 +12,7 @@ import ( logging "github.com/ipfs/go-log" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl" @@ -20,6 +21,8 @@ import ( func TestDealFlow(t *testing.T, b APIBuilder) { os.Setenv("BELLMAN_NO_GPU", "1") + build.SectorSizes = []uint64{1024} + logging.SetAllLoggers(logging.LevelInfo) ctx := context.Background() n, sn := b(t, 1, []int{0}) @@ -36,7 +39,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { } time.Sleep(time.Second) - r := io.LimitReader(rand.New(rand.NewSource(17)), 350) + r := io.LimitReader(rand.New(rand.NewSource(17)), 257) fcid, err := client.ClientImportLocal(ctx, r) if err != nil { t.Fatal(err) diff --git a/storage/miner.go b/storage/miner.go index c3ddcc8e5..eab867a43 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -38,9 +38,9 @@ type Miner struct { schedPost uint64 // Sealing - sb *sectorbuilder.SectorBuilder + sb *sectorbuilder.SectorBuilder sectors *statestore.StateStore - tktFn TicketFn + tktFn TicketFn sectorIncoming chan *SectorInfo sectorUpdated chan sectorUpdate @@ -80,7 +80,7 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto maddr: addr, h: h, - sb: sb, + sb: sb, tktFn: tktFn, sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))), diff --git a/storage/sealing.go b/storage/sealing.go index 4000d46fe..461728840 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -7,9 +7,9 @@ import ( cid "github.com/ipfs/go-cid" xerrors "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) @@ -27,9 +27,9 @@ func (t *SealTicket) sb() sectorbuilder.SealTicket { type Piece struct { DealID uint64 - Ref string + Ref string - Size uint64 + Size uint64 CommP []byte } @@ -52,7 +52,7 @@ type SectorInfo struct { CommD []byte CommR []byte CommRLast []byte - Ticket SealTicket + Ticket SealTicket PreCommitMessage *cid.Cid @@ -219,10 +219,10 @@ func (m *Miner) newSector(ctx context.Context, sid uint64, dealID uint64, ref st Pieces: []Piece{ { DealID: dealID, - Ref:ref, + Ref: ref, - Size: ppi.Size, - CommP: ppi.CommP[:], + Size: ppi.Size, + CommP: ppi.CommP[:], }, }, } diff --git a/storage/sector_states.go b/storage/sector_states.go index ded491192..dcf473b3b 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -43,11 +43,13 @@ func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*Sec allocated += piece.Size } - if allocated > m.sb.SectorSize() { - return nil, xerrors.Errorf("too much data in sector: %d > %d", allocated, m.sb.SectorSize()) + ubytes := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize()) + + if allocated > ubytes { + return nil, xerrors.Errorf("too much data in sector: %d > %d", allocated, ubytes) } - fillerSizes, err := fillersFromRem(m.sb.SectorSize() - allocated) + fillerSizes, err := fillersFromRem(ubytes - allocated) if err != nil { return nil, err } From f6a49ab9f97960ccf797fad35deb9018c74c0586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 20:54:24 +0100 Subject: [PATCH 187/230] Wip fixing a thing --- api/test/deals.go | 7 ++----- lib/sectorbuilder/files.go | 4 ++-- lib/sectorbuilder/sectorbuilder.go | 20 ++++++++++++++++++-- storage/garbage.go | 4 ++++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 8e8f37388..9312f5f13 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -12,7 +12,6 @@ import ( logging "github.com/ipfs/go-log" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/impl" @@ -21,8 +20,6 @@ import ( func TestDealFlow(t *testing.T, b APIBuilder) { os.Setenv("BELLMAN_NO_GPU", "1") - build.SectorSizes = []uint64{1024} - logging.SetAllLoggers(logging.LevelInfo) ctx := context.Background() n, sn := b(t, 1, []int{0}) @@ -39,7 +36,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { } time.Sleep(time.Second) - r := io.LimitReader(rand.New(rand.NewSource(17)), 257) + r := io.LimitReader(rand.New(rand.NewSource(17)), 9000000) fcid, err := client.ClientImportLocal(ctx, r) if err != nil { t.Fatal(err) @@ -65,7 +62,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { } } }() - deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(400), 100) + deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(40000000), 100) if err != nil { t.Fatal(err) } diff --git a/lib/sectorbuilder/files.go b/lib/sectorbuilder/files.go index c0af2b6a7..8a4500523 100644 --- a/lib/sectorbuilder/files.go +++ b/lib/sectorbuilder/files.go @@ -19,13 +19,13 @@ func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string { } func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) { - return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) } func (sb *SectorBuilder) sealedSectorPath(sectorID uint64) (string, error) { path := filepath.Join(sb.sealedDir, sb.sectorName(sectorID)) - e, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) + e, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return "", err } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 98e0dcfd1..d886db2d6 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -182,17 +182,33 @@ func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, piece return RawSealPreCommitOutput{}, err } - return sectorbuilder.StandaloneSealPreCommit( + var sum uint64 + for _, piece := range pieces { + sum += piece.Size + } + ussize := UserBytesForSectorSize(sb.ssize) + if sum != ussize { + return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize - sum)) + } + + stagedPath := sb.stagedSectorPath(sectorID) + + rspco, err := sectorbuilder.StandaloneSealPreCommit( sb.ssize, PoRepProofPartitions, cacheDir, - sb.stagedSectorPath(sectorID), + stagedPath, sealedPath, sectorID, addressToProverID(sb.Miner), ticket.TicketBytes, pieces, ) + if err != nil { + return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err) + } + + return rspco, nil } func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, pieceKeys []string, rspco RawSealPreCommitOutput) (proof []byte, err error) { diff --git a/storage/garbage.go b/storage/garbage.go index 8a2dd7a22..fda297f9c 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -18,6 +18,10 @@ import ( // TODO: expected sector ID func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, sizes ...uint64) ([]Piece, error) { + if len(sizes) == 0 { + return nil, nil + } + deals := make([]actors.StorageDeal, len(sizes)) for i, size := range sizes { commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) From 34846c538ee91120644c34b3290d324c4629fdbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 7 Nov 2019 21:40:46 +0100 Subject: [PATCH 188/230] sectorbuilder: Use StandaloneWriteWithAlignment --- api/test/deals.go | 2 +- lib/sectorbuilder/files.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 10 +++++----- lib/sectorbuilder/sectorbuilder_test.go | 2 +- storage/garbage.go | 9 +++++---- storage/sealing.go | 10 +++++++++- storage/sector_states.go | 2 +- 7 files changed, 23 insertions(+), 14 deletions(-) diff --git a/api/test/deals.go b/api/test/deals.go index 9312f5f13..cb3002459 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -36,7 +36,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) { } time.Sleep(time.Second) - r := io.LimitReader(rand.New(rand.NewSource(17)), 9000000) + r := io.LimitReader(rand.New(rand.NewSource(17)), 1000) fcid, err := client.ClientImportLocal(ctx, r) if err != nil { t.Fatal(err) diff --git a/lib/sectorbuilder/files.go b/lib/sectorbuilder/files.go index 8a4500523..51b748f50 100644 --- a/lib/sectorbuilder/files.go +++ b/lib/sectorbuilder/files.go @@ -19,7 +19,7 @@ func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string { } func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) { - return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) + return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644) } func (sb *SectorBuilder) sealedSectorPath(sectorID uint64) (string, error) { diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index d886db2d6..cc9bcdeea 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -124,7 +124,7 @@ func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { return sectorbuilder.AcquireSectorId(sb.handle) } -func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader) (PublicPieceInfo, error) { +func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) { f, werr, err := toReadableFile(file, int64(pieceSize)) if err != nil { return PublicPieceInfo{}, err @@ -138,13 +138,13 @@ func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Rea return PublicPieceInfo{}, err } - writeUnpadded, commP, err := sectorbuilder.StandaloneWriteWithoutAlignment(f, pieceSize, stagedFile) + _, _, commP, err := sectorbuilder.StandaloneWriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes) if err != nil { return PublicPieceInfo{}, err } - if writeUnpadded != pieceSize { + /*if writeUnpadded != pieceSize { return PublicPieceInfo{}, xerrors.Errorf("writeUnpadded != pieceSize: %d != %d", writeUnpadded, pieceSize) - } + }*/ if err := stagedFile.Close(); err != nil { return PublicPieceInfo{}, err @@ -188,7 +188,7 @@ func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, piece } ussize := UserBytesForSectorSize(sb.ssize) if sum != ussize { - return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize - sum)) + return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum)) } stagedPath := sb.stagedSectorPath(sectorID) diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 1b4933656..0e399be66 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -37,7 +37,7 @@ func TestSealAndVerify(t *testing.T) { } r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen)) - ppi, err := sb.AddPiece(dlen, sid, r) + ppi, err := sb.AddPiece(dlen, sid, r, []uint64{}) if err != nil { t.Fatalf("%+v", err) } diff --git a/storage/garbage.go b/storage/garbage.go index fda297f9c..28e33e77d 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -16,8 +16,7 @@ import ( "github.com/filecoin-project/lotus/lib/sectorbuilder" ) -// TODO: expected sector ID -func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, sizes ...uint64) ([]Piece, error) { +func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) { if len(sizes) == 0 { return nil, nil } @@ -95,11 +94,13 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, sizes ...uint for i, size := range sizes { name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) - ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size))) + ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes) if err != nil { return nil, err } + existingPieceSizes = append(existingPieceSizes, size) + out[i] = Piece{ DealID: resp.DealIDs[i], Ref: name, @@ -122,7 +123,7 @@ func (m *Miner) StoreGarbageData(_ context.Context) error { return } - pieces, err := m.storeGarbage(ctx, sid, size) + pieces, err := m.storeGarbage(ctx, sid, []uint64{}, size) if err != nil { log.Errorf("%+v", err) return diff --git a/storage/sealing.go b/storage/sealing.go index 461728840..de9586326 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -95,6 +95,14 @@ func (t *SectorInfo) refs() []string { return out } +func (t *SectorInfo) existingPieces() []uint64 { + out := make([]uint64, len(t.Pieces)) + for i, piece := range t.Pieces { + out[i] = piece.Size + } + return out +} + func (t *SectorInfo) rspco() sectorbuilder.RawSealPreCommitOutput { var out sectorbuilder.RawSealPreCommitOutput @@ -203,7 +211,7 @@ func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Rea return 0, xerrors.Errorf("acquiring sector ID: %w", err) } - ppi, err := m.sb.AddPiece(size, sid, r) + ppi, err := m.sb.AddPiece(size, sid, r, []uint64{}) if err != nil { return 0, xerrors.Errorf("adding piece to sector: %w", err) } diff --git a/storage/sector_states.go b/storage/sector_states.go index dcf473b3b..a567f4e65 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -58,7 +58,7 @@ func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*Sec log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID) } - pieces, err := m.storeGarbage(ctx, sector.SectorID, fillerSizes...) + pieces, err := m.storeGarbage(ctx, sector.SectorID, sector.existingPieces(), fillerSizes...) if err != nil { return nil, xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) } From af2789c3d88b2198c90cc5d67677d8fbf0490c5d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 7 Nov 2019 14:55:24 -0800 Subject: [PATCH 189/230] fix non-deterministic map serialization --- chain/actors/actor_miner.go | 4 +- chain/actors/actor_storagemarket.go | 6 ++- chain/actors/cbor_gen.go | 65 ++++++++++++++++++----------- chain/deals/client_states.go | 2 + chain/sync.go | 11 +++++ go.mod | 2 +- go.sum | 2 + 7 files changed, 64 insertions(+), 28 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 608cbd5df..d1e4ad4d3 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -305,7 +305,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.New(1, "no pre-commitment found for sector") } - if us.ReceivedEpoch+build.InteractivePoRepDelay > vmctx.BlockHeight() { + if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() { return nil, aerrors.New(2, "too early for proof submission") } @@ -340,7 +340,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { - return nil, aerrors.Newf(2, "bad proof! (t:%x; s:%x(%d); p:%x)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, params.Proof) + return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%x)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, params.Proof) } // Note: There must exist a unique index in the miner's sector set for each diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 4e00e970a..6fb07be3f 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -613,6 +613,10 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type return nil, aerrors.HandleExternalError(err, "getting deal info failed") } + if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + return nil, aerrors.New(4, "referenced deal was not from caller") + } + var commP [32]byte copy(commP[:], dealInfo.Deal.Proposal.PieceRef) @@ -624,7 +628,7 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces) if err != nil { - return nil, aerrors.Absorb(err, 4, "failed to generate data commitment from pieces") + return nil, aerrors.Absorb(err, 5, "failed to generate data commitment from pieces") } return commd[:], nil diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 7b5aef2fb..fbb289fd2 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3,6 +3,7 @@ package actors import ( "fmt" "io" + "sort" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -202,23 +203,31 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { } // t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map) - if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { - return err - } - - for k, v := range t.PreCommittedSectors { - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { - return err - } - if _, err := w.Write([]byte(k)); err != nil { + { + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil { return err } - if err := v.MarshalCBOR(w); err != nil { - return err + keys := make([]string, 0, len(t.PreCommittedSectors)) + for k := range t.PreCommittedSectors { + keys = append(keys, k) } + sort.Strings(keys) + for _, k := range keys { + v := t.PreCommittedSectors[k] + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + + } } // t.t.Sectors (cid.Cid) (struct) @@ -1902,23 +1911,31 @@ func (t *PaymentChannelActorState) MarshalCBOR(w io.Writer) error { } // t.t.LaneStates (map[string]*actors.LaneState) (map) - if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil { - return err - } - - for k, v := range t.LaneStates { - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { - return err - } - if _, err := w.Write([]byte(k)); err != nil { + { + if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil { return err } - if err := v.MarshalCBOR(w); err != nil { - return err + keys := make([]string, 0, len(t.LaneStates)) + for k := range t.LaneStates { + keys = append(keys, k) } + sort.Strings(keys) + for _, k := range keys { + v := t.LaneStates[k] + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil { + return err + } + if _, err := w.Write([]byte(k)); err != nil { + return err + } + + if err := v.MarshalCBOR(w); err != nil { + return err + } + + } } return nil } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 5b4b7c78c..26ac0ce68 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -3,6 +3,7 @@ package deals import ( "bytes" "context" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/lib/cborutil" @@ -97,6 +98,7 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea } if eq { dealIdx = i + break } } diff --git a/chain/sync.go b/chain/sync.go index 952b57a20..edaabc2a0 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -462,6 +462,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } if stateroot != h.ParentStateRoot { + msgs, err := syncer.store.MessagesForTipset(baseTs) + if err != nil { + log.Error("failed to load messages for tipset during tipset state mismatch error: ", err) + } else { + log.Warn("Messages for tipset with mismatching state:") + for i, m := range msgs { + mm := m.VMMessage() + log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params) + } + } + return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot) } diff --git a/go.mod b/go.mod index da117bec0..8e62267a7 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f + github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.0 diff --git a/go.sum b/go.sum index d02fd9826..665d7d4be 100644 --- a/go.sum +++ b/go.sum @@ -530,6 +530,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:x github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f h1:+GFA37QICd1Axd2n9uzjtvPjxJJI5PU78vpvam+hI4U= github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 h1:ct50KYdZHcdOnTAuSgppw5MZKTa3RA63FX28m0l9Aeg= +github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= From eae0b84b801e85b066978c46444432c46ebe816b Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 7 Nov 2019 17:45:45 -0800 Subject: [PATCH 190/230] Add command to list client deals --- cli/client.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cli/client.go b/cli/client.go index 2ac4b3db3..283de413e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -2,14 +2,17 @@ package cli import ( "fmt" + "os" "path/filepath" "strconv" + "text/tabwriter" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + lapi "github.com/filecoin-project/lotus/api" actors "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" @@ -25,6 +28,7 @@ var clientCmd = &cli.Command{ clientFindCmd, clientRetrieveCmd, clientQueryAskCmd, + clientListDeals, }, } @@ -326,3 +330,28 @@ var clientQueryAskCmd = &cli.Command{ return nil }, } + +var clientListDeals = &cli.Command{ + Name: "list-deals", + Usage: "List storage market deals", + Action: func(cctx *cli.Context) error { + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := ReqContext(cctx) + + deals, err := api.ClientListDeals(ctx) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "DealCid\tProvider\tState\tPieceRef\tSize\tPrice\tDuration\n") + for _, d := range deals { + fmt.Fprintf(w, "%s\t%s\t%s\t%x\t%d\t%s\t%d\n", d.ProposalCid, d.Provider, lapi.DealStates[d.State], d.PieceRef, d.Size, d.PricePerEpoch, d.Duration) + } + return w.Flush() + }, +} From 721786da1048736023e499d0f57d4644d6f6c7c9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 7 Nov 2019 22:06:08 -0800 Subject: [PATCH 191/230] add a flag to allow only fetching verifying keys --- cli/params.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/params.go b/cli/params.go index 6f6017ca3..9e8183b8a 100644 --- a/cli/params.go +++ b/cli/params.go @@ -9,8 +9,14 @@ import ( var fetchParamCmd = &cli.Command{ Name: "fetch-params", Usage: "Fetch proving parameters", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "only-verify-keys", + Usage: "only download the verify keys", + }, + }, Action: func(cctx *cli.Context) error { - if err := build.GetParams(true); err != nil { + if err := build.GetParams(!cctx.Bool("only-verify-keys")); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } From 2560c091da716dc521403ed98d38fe7d33573b25 Mon Sep 17 00:00:00 2001 From: wanghui Date: Fri, 8 Nov 2019 16:40:02 +0800 Subject: [PATCH 192/230] make long number param readable --- build/params.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/params.go b/build/params.go index e810ff45f..aaa7b475e 100644 --- a/build/params.go +++ b/build/params.go @@ -84,14 +84,14 @@ const CollateralPrecision = 1000 // ///// // Devnet settings -const TotalFilecoin = 2000000000 -const MiningRewardTotal = 1400000000 +const TotalFilecoin = 2_000_000_000 +const MiningRewardTotal = 1_400_000_000 const InitialRewardStr = "153856861913558700202" var InitialReward *big.Int -const FilecoinPrecision = 1000000000000000000 +const FilecoinPrecision = 1_000_000_000_000_000_000 // six years // Blocks From e226071c2036d679bceaf68c170c8a78b78ba5a5 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 8 Nov 2019 08:43:23 -0800 Subject: [PATCH 193/230] disable GPU by default --- cmd/lotus-storage-miner/run.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 4142233fd..c8a10baac 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -31,12 +31,20 @@ var runCmd = &cli.Command{ Name: "api", Value: "2345", }, + &cli.BoolFlag{ + Name: "enable-gpu-proving", + Usage: "Enable use of GPU for mining operations", + }, }, Action: func(cctx *cli.Context) error { if err := build.GetParams(true); err != nil { return xerrors.Errorf("fetching proof parameters: %w", err) } + if !cctx.Bool("enable-gpu-proving") { + os.Setenv("BELLMAN_NO_GPU", "true") + } + nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err From be2e58a2fb47f4586a6141d9664086d05a45cb6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 18:15:38 +0100 Subject: [PATCH 194/230] Simple market fund manager --- api/api_full.go | 3 ++ api/struct.go | 6 +++ chain/deals/client.go | 45 ++++---------------- chain/deals/provider.go | 2 +- chain/deals/provider_states.go | 41 ++---------------- chain/market/fundmgr.go | 78 ++++++++++++++++++++++++++++++++++ node/builder.go | 2 + node/impl/full.go | 2 + node/impl/market/market.go | 21 +++++++++ 9 files changed, 125 insertions(+), 75 deletions(-) create mode 100644 chain/market/fundmgr.go create mode 100644 node/impl/market/market.go diff --git a/api/api_full.go b/api/api_full.go index d46fd25ba..ecafaadc1 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -111,6 +111,9 @@ type FullNode interface { StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) + MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error + // MarketFreeBalance + PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) PaychList(context.Context) ([]address.Address, error) PaychStatus(context.Context, address.Address) (*PaychStatus, error) diff --git a/api/struct.go b/api/struct.go index 682e8f3e7..2845e05c7 100644 --- a/api/struct.go +++ b/api/struct.go @@ -107,6 +107,8 @@ type FullNodeStruct struct { StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"` StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"` + MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` + PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` PaychList func(context.Context) ([]address.Address, error) `perm:"read"` PaychStatus func(context.Context, address.Address) (*PaychStatus, error) `perm:"read"` @@ -418,6 +420,10 @@ func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid uint return c.Internal.StateMarketStorageDeal(ctx, dealid, ts) } +func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + return c.Internal.MarketEnsureAvailable(ctx, addr, amt) +} + func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } diff --git a/chain/deals/client.go b/chain/deals/client.go index f1101b1e3..eb96be154 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -3,9 +3,6 @@ package deals import ( "context" - "github.com/filecoin-project/lotus/lib/statestore" - "github.com/filecoin-project/lotus/node/impl/full" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -19,11 +16,14 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/events" + "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/lib/statestore" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrieval/discovery" ) @@ -50,8 +50,8 @@ type Client struct { w *wallet.Wallet dag dtypes.ClientDAG discovery *discovery.Local - mpool full.MpoolAPI events *events.Events + fm *market.FundMgr deals *statestore.StateStore conns map[cid.Cid]inet.Stream @@ -70,7 +70,7 @@ type clientDealUpdate struct { mut func(*ClientDeal) } -func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI, chainapi full.ChainAPI) *Client { +func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, fm *market.FundMgr, chainapi full.ChainAPI) *Client { c := &Client{ sm: sm, chain: chain, @@ -78,7 +78,7 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w * w: w, dag: dag, discovery: discovery, - mpool: mpool, + fm: fm, events: events.NewEvents(context.TODO(), &chainapi), deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))), @@ -137,7 +137,7 @@ func (c *Client) onIncoming(deal *ClientDeal) { } func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { - log.Infof("Deal %s updated state to %d", update.id, update.newState) + log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState]) var deal ClientDeal err := c.deals.Mutate(update.id, func(d *ClientDeal) error { d.State = update.newState @@ -184,35 +184,8 @@ type ClientDealProposal struct { } func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) { - // check market funds - clientMarketBalance, err := c.sm.MarketBalance(ctx, p.Client, nil) - if err != nil { - return cid.Undef, xerrors.Errorf("getting client market balance failed: %w", err) - } - - if clientMarketBalance.Available.LessThan(types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))) { - // TODO: move to a smarter market funds manager - - smsg, err := c.mpool.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: p.Client, - Value: types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.AddBalance, - }) - if err != nil { - return cid.Undef, err - } - - _, r, err := c.sm.WaitForMessage(ctx, smsg.Cid()) - if err != nil { - return cid.Undef, err - } - - if r.ExitCode != 0 { - return cid.Undef, xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode) - } + if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil { + return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } commP, pieceSize, err := c.commP(ctx, p.Data) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index 8430cf2f4..2c8978170 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -160,7 +160,7 @@ func (p *Provider) onIncoming(deal MinerDeal) { } func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) { - log.Infof("Deal %s updated state to %d", update.id, update.newState) + log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState]) if update.err != nil { log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err) p.failDeal(update.id, update.err) diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 00546202d..35f339502 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -10,7 +10,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/storage/sectorblocks" @@ -39,33 +38,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle } // ACCEPTED - -func (p *Provider) addMarketFunds(ctx context.Context, worker address.Address, deal MinerDeal) error { - log.Info("Adding market funds for storage collateral") - smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: worker, - Value: deal.Proposal.StorageCollateral, - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: actors.SMAMethods.AddBalance, - }) - if err != nil { - return err - } - - r, err := p.full.StateWaitMsg(ctx, smsg.Cid()) - if err != nil { - return err - } - - if r.Receipt.ExitCode != 0 { - return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode) - } - - return nil -} - func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { switch deal.Proposal.PieceSerialization { //case SerializationRaw: @@ -111,16 +83,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) return nil, err } - providerMarketBalance, err := p.full.StateMarketBalance(ctx, waddr, nil) - if err != nil { - return nil, xerrors.Errorf("getting provider market balance failed: %w", err) - } - - // TODO: this needs to be atomic - if providerMarketBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - if err := p.addMarketFunds(ctx, waddr, deal); err != nil { - return nil, err - } + // TODO: check StorageCollateral (may be too large (or too small)) + if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil { + return nil, err } log.Info("publishing deal") diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go new file mode 100644 index 000000000..05716a428 --- /dev/null +++ b/chain/market/fundmgr.go @@ -0,0 +1,78 @@ +package market + +import ( + "context" + "sync" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/impl/full" +) + +type FundMgr struct { + sm *stmgr.StateManager + mpool full.MpoolAPI + + lk sync.Mutex + available map[address.Address]types.BigInt +} + +func NewFundMgr(sm *stmgr.StateManager, mpool full.MpoolAPI) *FundMgr { + return &FundMgr{ + sm: sm, + mpool: mpool, + + available: map[address.Address]types.BigInt{}, + } +} + +func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + fm.lk.Lock() + avail, ok := fm.available[addr] + if !ok { + bal, err := fm.sm.MarketBalance(ctx, addr, nil) + if err != nil { + fm.lk.Unlock() + return err + } + + avail = bal.Available + } + + toAdd := types.NewInt(0) + avail = types.BigSub(avail, amt) + if avail.LessThan(types.NewInt(0)) { + // TODO: some rules around adding more to avoid doing stuff on-chain + // all the time + toAdd = types.BigSub(toAdd, avail) + avail = types.NewInt(0) + } + fm.available[addr] = avail + fm.lk.Unlock() + + smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{ + To: actors.StorageMarketAddress, + From: addr, + Value: toAdd, + GasPrice: types.NewInt(0), + GasLimit: types.NewInt(1000000), + Method: actors.SMAMethods.AddBalance, + }) + if err != nil { + return err + } + + _, r, err := fm.sm.WaitForMessage(ctx, smsg.Cid()) + if err != nil { + return err + } + + if r.ExitCode != 0 { + return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode) + } + return nil +} diff --git a/node/builder.go b/node/builder.go index cf999ea4b..147366db8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "github.com/filecoin-project/lotus/chain/market" "time" blockstore "github.com/ipfs/go-ipfs-blockstore" @@ -221,6 +222,7 @@ func Online() Option { Override(new(*paych.Store), paych.NewStore), Override(new(*paych.Manager), paych.NewManager), + Override(new(*market.FundMgr), market.NewFundMgr), Override(new(*miner.Miner), miner.NewMiner), ), diff --git a/node/impl/full.go b/node/impl/full.go index 41c0730bf..6ce7fa91e 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -2,6 +2,7 @@ package impl import ( "context" + "github.com/filecoin-project/lotus/node/impl/market" "github.com/filecoin-project/lotus/node/impl/client" "github.com/filecoin-project/lotus/node/impl/paych" @@ -21,6 +22,7 @@ type FullNodeAPI struct { full.ChainAPI client.API full.MpoolAPI + market.MarketAPI paych.PaychAPI full.StateAPI full.WalletAPI diff --git a/node/impl/market/market.go b/node/impl/market/market.go new file mode 100644 index 000000000..3e7724e43 --- /dev/null +++ b/node/impl/market/market.go @@ -0,0 +1,21 @@ +package market + +import ( + "context" + + "go.uber.org/fx" + + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/market" + "github.com/filecoin-project/lotus/chain/types" +) + +type MarketAPI struct { + fx.In + + FMgr *market.FundMgr +} + +func (a *MarketAPI) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error { + return a.FMgr.EnsureAvailable(ctx, addr, amt) +} From 071f05fa955a068a227a862de83b00054a445b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 19:15:13 +0100 Subject: [PATCH 195/230] Storage miner API improvements --- api/api_full.go | 6 +- api/api_storage.go | 21 ++++- api/struct.go | 22 +++-- build/version.go | 4 +- chain/stmgr/utils.go | 10 +-- cmd/lotus-storage-miner/info.go | 15 ++-- cmd/lotus-storage-miner/sectors.go | 8 +- gen/main.go | 1 + lib/sectorbuilder/sectorbuilder.go | 4 + node/impl/full/state.go | 4 +- node/impl/storminer.go | 43 +++++++++- storage/cbor_gen.go | 130 ++++++++++++++++++++++------- storage/miner.go | 4 +- storage/post.go | 4 +- storage/sealing.go | 18 +++- storage/sealing_utils.go | 14 ++++ storage/sector_states.go | 28 +++---- 17 files changed, 247 insertions(+), 89 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index ecafaadc1..057dca8cb 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -95,8 +95,8 @@ type FullNode interface { StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) - StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) - StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) + StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) StateMinerPower(context.Context, address.Address, *types.TipSet) (MinerPower, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) @@ -164,7 +164,7 @@ type Message struct { Message *types.Message } -type SectorInfo struct { +type ChainSectorInfo struct { SectorID uint64 CommD []byte CommR []byte diff --git a/api/api_storage.go b/api/api_storage.go index 75fc55849..299449662 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -58,12 +58,31 @@ type StorageMiner interface { StoreGarbageData(context.Context) error // Get the status of a given sector by ID - SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) + SectorsStatus(context.Context, uint64) (SectorInfo, error) // List all staged sectors SectorsList(context.Context) ([]uint64, error) SectorsRefs(context.Context) (map[string][]SealedRef, error) + + WorkerStats(context.Context) (WorkerStats, error) +} + +type WorkerStats struct { + Free int + Reserved int // for PoSt + Total int +} + +type SectorInfo struct { + SectorID uint64 + State SectorState + CommD []byte + CommR []byte + Proof []byte + Deals []uint64 + Ticket sectorbuilder.SealTicket + Seed sectorbuilder.SealSeed } type SealedRef struct { diff --git a/api/struct.go b/api/struct.go index 2845e05c7..8d14af1d2 100644 --- a/api/struct.go +++ b/api/struct.go @@ -3,7 +3,6 @@ package api import ( "context" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -87,8 +86,8 @@ type FullNodeStruct struct { ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` @@ -133,10 +132,11 @@ type StorageMinerStruct struct { StoreGarbageData func(context.Context) error `perm:"write"` - SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` - SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsStatus func(context.Context, uint64) (SectorInfo, error) `perm:"read"` + SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` + WorkerStats func(context.Context) (WorkerStats, error) `perm:"read"` } } @@ -345,11 +345,11 @@ func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMs return c.Internal.SyncSubmitBlock(ctx, blk) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) { +func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { return c.Internal.StateMinerProvingSet(ctx, addr, ts) } @@ -481,7 +481,7 @@ func (c *StorageMinerStruct) StoreGarbageData(ctx context.Context) error { } // Get the status of a given sector by ID -func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { +func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) { return c.Internal.SectorsStatus(ctx, sid) } @@ -494,6 +494,10 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]Seal return c.Internal.SectorsRefs(ctx) } +func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (WorkerStats, error) { + return c.Internal.WorkerStats(ctx) +} + var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/build/version.go b/build/version.go index 33109762d..abc715b48 100644 --- a/build/version.go +++ b/build/version.go @@ -1,7 +1,7 @@ package build // Version is the local build version, set by build system -const Version = "0.0.0" +const Version = "0.7.0" // APIVersion is a hex semver version of the rpc api exposed // @@ -12,7 +12,7 @@ const Version = "0.0.0" // R R H // |\vv/| // vv vv -const APIVersion = 0x000001 +const APIVersion = 0x000701 const ( MajorMask = 0xff0000 diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index ecd5acf87..7a89ab8d5 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -157,7 +157,7 @@ func GetMinerProvingPeriodEnd(ctx context.Context, sm *StateManager, ts *types.T return mas.ProvingPeriodEnd, nil } -func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) { +func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) { var mas actors.StorageMinerActorState _, err := sm.LoadActorState(ctx, maddr, &mas, ts) if err != nil { @@ -167,7 +167,7 @@ func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.ProvingSet) } -func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) { +func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) { var mas actors.StorageMinerActorState _, err := sm.LoadActorState(ctx, maddr, &mas, ts) if err != nil { @@ -213,20 +213,20 @@ func GetStorageDeal(ctx context.Context, sm *StateManager, dealId uint64, ts *ty return &ocd, nil } -func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) { +func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.ChainSectorInfo, error) { blks := amt.WrapBlockstore(bs) a, err := amt.LoadAMT(blks, ssc) if err != nil { return nil, err } - var sset []*api.SectorInfo + var sset []*api.ChainSectorInfo if err := a.ForEach(func(i uint64, v *cbg.Deferred) error { var comms [][]byte if err := cbor.DecodeInto(v.Raw, &comms); err != nil { return err } - sset = append(sset, &api.SectorInfo{ + sset = append(sset, &api.ChainSectorInfo{ SectorID: i, CommR: comms[0], CommD: comms[1], diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 213204a99..00864ed58 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -44,17 +44,18 @@ var infoCmd = &cli.Command{ percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower) fmt.Printf("Power: %s / %s (%0.2f%%)\n", pow.MinerPower, pow.TotalPower, float64(percI.Int64())/1000*100) + // TODO: indicate whether the post worker is in use + wstat, err := nodeApi.WorkerStats(ctx) + if err != nil { + return err + } + fmt.Printf("Worker use: %d / %d (+%d)", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved) + sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { return err } - /* - fmt.Println("Sealed Sectors:\t", sinfo.SealedCount) - fmt.Println("Sealing Sectors:\t", sinfo.SealingCount) - fmt.Println("Pending Sectors:\t", sinfo.PendingCount) - fmt.Println("Failed Sectors:\t", sinfo.FailedCount) - */ fmt.Println(sinfo) // TODO: grab actr state / info @@ -80,7 +81,7 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) (map[string]int, er return nil, err } - out[st.State.String()]++ + out[api.SectorStateStr(st.State)]++ } return out, nil diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index d49384619..b9075e443 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/filecoin-project/lotus/api" "strconv" "gopkg.in/urfave/cli.v2" @@ -60,14 +61,15 @@ var sectorsStatusCmd = &cli.Command{ } fmt.Printf("SectorID:\t%d\n", status.SectorID) - fmt.Printf("Status:\t%s\n", status.State.String()) - fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg) + fmt.Printf("Status:\t%s\n", api.SectorStateStr(status.State)) fmt.Printf("CommD:\t\t%x\n", status.CommD) fmt.Printf("CommR:\t\t%x\n", status.CommR) fmt.Printf("Ticket:\t\t%x\n", status.Ticket.TicketBytes) fmt.Printf("TicketH:\t\t%d\n", status.Ticket.BlockHeight) + fmt.Printf("Seed:\t\t%x\n", status.Seed.TicketBytes) + fmt.Printf("SeedH:\t\t%d\n", status.Seed.BlockHeight) fmt.Printf("Proof:\t\t%x\n", status.Proof) - fmt.Printf("Pieces:\t\t%v\n", status.Pieces) + fmt.Printf("Deals:\t\t%v\n", status.Deals) return nil }, } diff --git a/gen/main.go b/gen/main.go index 2c9292283..b2b0140e7 100644 --- a/gen/main.go +++ b/gen/main.go @@ -152,6 +152,7 @@ func main() { err = gen.WriteTupleEncodersToFile("./storage/cbor_gen.go", "storage", storage.SealTicket{}, + storage.SealSeed{}, storage.Piece{}, storage.SectorInfo{}, ) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index cc9bcdeea..27b804015 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -110,6 +110,10 @@ func (sb *SectorBuilder) rlimit() func() { } } +func (sb *SectorBuilder) WorkerStats() (free, reserved, total int) { + return cap(sb.rateLimit) - len(sb.rateLimit), PoStReservedWorkers, cap(sb.rateLimit) + PoStReservedWorkers +} + func addressToProverID(a address.Address) [32]byte { var proverId [32]byte copy(proverId[:], a.Payload()) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 606e3416a..93bc973c1 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -38,11 +38,11 @@ type StateAPI struct { Chain *store.ChainStore } -func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { +func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr) } -func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) { +func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return stmgr.GetMinerProvingSet(ctx, a.StateManager, ts, addr) } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index e7b081ea6..49c807412 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -21,6 +21,15 @@ type StorageMinerAPI struct { Full api.FullNode } +func (sm *StorageMinerAPI) WorkerStats(context.Context) (api.WorkerStats, error) { + free, reserved, total := sm.SectorBuilder.WorkerStats() + return api.WorkerStats{ + Free: free, + Reserved: reserved, + Total: total, + }, nil +} + func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { return sm.SectorBuilderConfig.Miner, nil } @@ -29,13 +38,41 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { return sm.Miner.StoreGarbageData(ctx) } -func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { - return sm.SectorBuilder.SealStatus(sid) +func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) { + info, err := sm.Miner.GetSectorInfo(sid) + if err != nil { + return api.SectorInfo{}, err + } + + deals := make([]uint64, len(info.Pieces)) + for i, piece := range info.Pieces { + deals[i] = piece.DealID + } + + return api.SectorInfo{ + SectorID: sid, + State: info.State, + CommD: info.CommD, + CommR: info.CommR, + Proof: info.Proof, + Deals: deals, + Ticket: info.Ticket.SB(), + Seed: info.Seed.SB(), + }, nil } // List all staged sectors func (sm *StorageMinerAPI) SectorsList(context.Context) ([]uint64, error) { - return sm.SectorBuilder.GetAllStagedSectors() + sectors, err := sm.Miner.ListSectors() + if err != nil { + return nil, err + } + + out := make([]uint64, len(sectors)) + for i, sector := range sectors { + out[i] = sector.SectorID + } + return out, nil } func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index c84a32590..623cd31a9 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -4,7 +4,6 @@ import ( "fmt" "io" - "github.com/filecoin-project/lotus/chain/types" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -82,6 +81,75 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *SealSeed) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.t.BlockHeight (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil { + return err + } + + // t.t.TicketBytes ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil { + return err + } + if _, err := w.Write(t.TicketBytes); err != nil { + return err + } + return nil +} + +func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.BlockHeight (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = uint64(extra) + // t.t.TicketBytes ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + return nil +} + func (t *Piece) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -245,6 +313,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } + // t.t.Proof ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + // t.t.Ticket (storage.SealTicket) (struct) if err := t.Ticket.MarshalCBOR(w); err != nil { return err @@ -262,13 +338,8 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } - // t.t.RandHeight (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RandHeight))); err != nil { - return err - } - - // t.t.RandTs (types.TipSet) (struct) - if err := t.RandTs.MarshalCBOR(w); err != nil { + // t.t.Seed (storage.SealSeed) (struct) + if err := t.Seed.MarshalCBOR(w); err != nil { return err } @@ -416,6 +487,23 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if _, err := io.ReadFull(br, t.CommRLast); err != nil { return err } + // t.t.Proof ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.Proof: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } // t.t.Ticket (storage.SealTicket) (struct) { @@ -449,35 +537,13 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } } - // t.t.RandHeight (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.RandHeight = uint64(extra) - // t.t.RandTs (types.TipSet) (struct) + // t.t.Seed (storage.SealSeed) (struct) { - pb, err := br.PeekByte() - if err != nil { + if err := t.Seed.UnmarshalCBOR(br); err != nil { return err } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.RandTs = new(types.TipSet) - if err := t.RandTs.UnmarshalCBOR(br); err != nil { - return err - } - } } // t.t.CommitMessage (cid.Cid) (struct) diff --git a/storage/miner.go b/storage/miner.go index eab867a43..c956fd115 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -56,8 +56,8 @@ type storageMinerApi interface { StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error) StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error) - StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) - StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error) + StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) + StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error) StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) diff --git a/storage/post.go b/storage/post.go index 0f4c74f38..13a635c0d 100644 --- a/storage/post.go +++ b/storage/post.go @@ -107,7 +107,7 @@ type post struct { ts *types.TipSet // prep - sset []*api.SectorInfo + sset []*api.ChainSectorInfo r []byte // run @@ -277,7 +277,7 @@ func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipS } } -func sectorIdList(si []*api.SectorInfo) []uint64 { +func sectorIdList(si []*api.ChainSectorInfo) []uint64 { out := make([]uint64, len(si)) for i, s := range si { out[i] = s.SectorID diff --git a/storage/sealing.go b/storage/sealing.go index de9586326..232ddf5c4 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -8,7 +8,6 @@ import ( xerrors "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sectorbuilder" ) @@ -19,12 +18,23 @@ type SealTicket struct { TicketBytes []byte } -func (t *SealTicket) sb() sectorbuilder.SealTicket { +func (t *SealTicket) SB() sectorbuilder.SealTicket { out := sectorbuilder.SealTicket{BlockHeight: t.BlockHeight} copy(out.TicketBytes[:], t.TicketBytes) return out } +type SealSeed struct { + BlockHeight uint64 + TicketBytes []byte +} + +func (t *SealSeed) SB() sectorbuilder.SealSeed { + out := sectorbuilder.SealSeed{BlockHeight: t.BlockHeight} + copy(out.TicketBytes[:], t.TicketBytes) + return out +} + type Piece struct { DealID uint64 Ref string @@ -52,13 +62,13 @@ type SectorInfo struct { CommD []byte CommR []byte CommRLast []byte + Proof []byte Ticket SealTicket PreCommitMessage *cid.Cid // PreCommitted - RandHeight uint64 - RandTs *types.TipSet + Seed SealSeed // Committing CommitMessage *cid.Cid diff --git a/storage/sealing_utils.go b/storage/sealing_utils.go index 17d83abbf..26591e0ac 100644 --- a/storage/sealing_utils.go +++ b/storage/sealing_utils.go @@ -18,3 +18,17 @@ func fillersFromRem(toFill uint64) ([]uint64, error) { } return out, nil } + +func (m *Miner) ListSectors() ([]SectorInfo, error) { + var sectors []SectorInfo + if err := m.sectors.List(§ors); err != nil { + return nil, err + } + return sectors, nil +} + +func (m *Miner) GetSectorInfo(sid uint64) (SectorInfo, error) { + var out SectorInfo + err := m.sectors.Get(sid, &out) + return out, err +} diff --git a/storage/sector_states.go b/storage/sector_states.go index a567f4e65..f94b06072 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -145,12 +145,19 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight) err = m.events.ChainAt(func(ctx context.Context, ts *types.TipSet, curH uint64) error { + rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-randHeight)) + if err != nil { + return xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) + } + m.sectorUpdated <- sectorUpdate{ newState: api.Committing, id: sector.SectorID, mut: func(info *SectorInfo) { - info.RandHeight = randHeight - info.RandTs = ts + info.Seed = SealSeed{ + BlockHeight: randHeight, + TicketBytes: rand, + } }, } @@ -169,21 +176,13 @@ func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*Sect func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) { log.Info("scheduling seal proof computation...") - rand, err := m.api.ChainGetRandomness(ctx, sector.RandTs, nil, int(sector.RandTs.Height()-sector.RandHeight)) - if err != nil { - return nil, xerrors.Errorf("failed to get randomness for computing seal proof: %w", err) - } - - seed := sectorbuilder.SealSeed{ - BlockHeight: sector.RandHeight, - } - copy(seed.TicketBytes[:], rand) - - proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.sb(), seed, sector.pieceInfos(), sector.refs(), sector.rspco()) + proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.refs(), sector.rspco()) if err != nil { return nil, xerrors.Errorf("computing seal proof failed: %w", err) } + // TODO: Consider splitting states and persist proof for faster recovery + params := &actors.SectorProveCommitInfo{ Proof: proof, SectorID: sector.SectorID, @@ -218,7 +217,7 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector } if mw.Receipt.ExitCode != 0 { - log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, rand, sector.RandHeight, params.Proof) + log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, params.Proof) return nil, xerrors.New("UNHANDLED: submitting sector proof failed") } @@ -227,5 +226,6 @@ func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*Sector return func(info *SectorInfo) { mcid := smsg.Cid() info.CommitMessage = &mcid + info.Proof = proof }, nil } From 77dbb6765144229adbdb01bd9eb4c1e541a2b03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 19:26:04 +0100 Subject: [PATCH 196/230] pond: Update sealing status ids --- lotuspond/front/src/StorageNode.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lotuspond/front/src/StorageNode.js b/lotuspond/front/src/StorageNode.js index ef50c615d..c07abc590 100644 --- a/lotuspond/front/src/StorageNode.js +++ b/lotuspond/front/src/StorageNode.js @@ -8,16 +8,14 @@ const stateConnecting = 'connecting' const stateGettingToken = 'getting-token' let sealCodes = [ - "Unknown", - "AcceptingPieces", - "Committed", - "Committing", - "CommittingPaused", - "Failed", - "FullyPacked", - "PreCommitted", + "Undefined", + "Empty", + "Packing", + "Unsealed", "PreCommitting", - "PreCommittingPaused", + "PreCommitted", + "Committing", + "Proving", ] class StorageNode extends React.Component { From c729cabef156fdd6e9085e41ee0566b3e1e580d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 19:49:36 +0100 Subject: [PATCH 197/230] keep track of last used sector id across restarts --- lib/sectorbuilder/mock.go | 7 ++- lib/sectorbuilder/sectorbuilder.go | 60 ++++++++++++++++++++++--- lib/sectorbuilder/sectorbuilder_test.go | 41 +++++++++++++++-- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 20ec4ac51..58db95d09 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -6,9 +6,10 @@ import ( "path/filepath" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) -func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { +func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, func(), error) { dir, err := ioutil.TempDir("", "sbtest") if err != nil { return nil, nil, err @@ -34,12 +35,14 @@ func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) { WorkerThreads: 2, Miner: addr, - }) + }, ds, nil) if err != nil { return nil, nil, err } return sb, func() { + sb.Destroy() + if err := os.RemoveAll(dir); err != nil { log.Warn("failed to clean up temp sectorbuilder: ", err) } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 27b804015..bdc8d5a66 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -1,21 +1,30 @@ package sectorbuilder import ( + "context" + "fmt" "io" "os" "sort" + "strconv" + "sync" "unsafe" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" + "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/node/modules/dtypes" ) const PoStReservedWorkers = 1 const PoRepProofPartitions = 2 +var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last") + var log = logging.Logger("sectorbuilder") type SectorSealingStatus = sectorbuilder.SectorSealingStatus @@ -42,6 +51,9 @@ const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer + ds dtypes.MetadataDS + idLk sync.Mutex + ssize uint64 Miner address.Address @@ -65,7 +77,7 @@ type Config struct { MetadataDir string } -func New(cfg *Config) (*SectorBuilder, error) { +func New(cfg *Config, ds dtypes.MetadataDS, lc fx.Lifecycle) (*SectorBuilder, error) { if cfg.WorkerThreads <= PoStReservedWorkers { return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads) } @@ -81,13 +93,29 @@ func New(cfg *Config) (*SectorBuilder, error) { } } - sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) + var lastUsedID uint64 + b, err := ds.Get(lastSectorIdKey) + switch err { + case nil: + i, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return nil, err + } + lastUsedID = uint64(i) + case datastore.ErrNotFound: + default: + return nil, err + } + + sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, lastUsedID, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads) if err != nil { return nil, err } - return &SectorBuilder{ + sb := &SectorBuilder{ handle: sbp, + ds: ds, + ssize: cfg.SectorSize, stagedDir: cfg.StagedDir, @@ -96,7 +124,18 @@ func New(cfg *Config) (*SectorBuilder, error) { Miner: cfg.Miner, rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), - }, nil + } + + if lc != nil { + lc.Append(fx.Hook{ + OnStop: func(context.Context) error { + sb.Destroy() + return nil + }, + }) + } + + return sb, nil } func (sb *SectorBuilder) rlimit() func() { @@ -125,7 +164,18 @@ func (sb *SectorBuilder) Destroy() { } func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { - return sectorbuilder.AcquireSectorId(sb.handle) + sb.idLk.Lock() + defer sb.idLk.Unlock() + + id, err := sectorbuilder.AcquireSectorId(sb.handle) + if err != nil { + return 0, err + } + err = sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))) + if err != nil { + return 0, err + } + return id, nil } func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) { diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 0e399be66..138cd939c 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -6,6 +6,10 @@ import ( "os" "testing" + "github.com/ipfs/go-datastore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/lib/sectorbuilder" ) @@ -22,12 +26,11 @@ func TestSealAndVerify(t *testing.T) { t.Fatalf("%+v", err) } - sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize) + sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, datastore.NewMapDatastore()) if err != nil { t.Fatalf("%+v", err) } - _ = cleanup - //defer cleanup() + defer cleanup() dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) @@ -91,3 +94,35 @@ func TestSealAndVerify(t *testing.T) { t.Fatal("bad post") } } + +func TestAcquireID(t *testing.T) { + ds := datastore.NewMapDatastore() + + sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, ds) + if err != nil { + t.Fatalf("%+v", err) + } + + assertAcquire := func(expect uint64) { + id, err := sb.AcquireSectorId() + require.NoError(t, err) + assert.Equal(t, expect, id) + } + + assertAcquire(1) + assertAcquire(2) + assertAcquire(3) + + cleanup() + + sb, cleanup, err = sectorbuilder.TempSectorbuilder(sectorSize, ds) + if err != nil { + t.Fatalf("%+v", err) + } + + assertAcquire(4) + assertAcquire(5) + assertAcquire(6) + + cleanup() +} From 61505b68022dd89ac0ebf403bee3ad4796b34dcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 19:55:23 +0100 Subject: [PATCH 198/230] Use the same dir in TestAcquireID --- lib/sectorbuilder/mock.go | 23 ++++++++++++++--------- lib/sectorbuilder/sectorbuilder_test.go | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 58db95d09..481b982b4 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -15,9 +15,20 @@ func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, return nil, nil, err } + sb, err := TempSectorbuilderDir(dir, sectorSize, ds) + return sb, func() { + sb.Destroy() + + if err := os.RemoveAll(dir); err != nil { + log.Warn("failed to clean up temp sectorbuilder: ", err) + } + }, err +} + +func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) { addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") if err != nil { - return nil, nil, err + return nil, err } metadata := filepath.Join(dir, "meta") @@ -37,14 +48,8 @@ func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, Miner: addr, }, ds, nil) if err != nil { - return nil, nil, err + return nil, err } - return sb, func() { - sb.Destroy() - - if err := os.RemoveAll(dir); err != nil { - log.Warn("failed to clean up temp sectorbuilder: ", err) - } - }, nil + return sb, nil } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 138cd939c..e02368913 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -2,6 +2,7 @@ package sectorbuilder_test import ( "io" + "io/ioutil" "math/rand" "os" "testing" @@ -98,7 +99,12 @@ func TestSealAndVerify(t *testing.T) { func TestAcquireID(t *testing.T) { ds := datastore.NewMapDatastore() - sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, ds) + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + t.Fatal(err) + } + + sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) if err != nil { t.Fatalf("%+v", err) } @@ -113,9 +119,9 @@ func TestAcquireID(t *testing.T) { assertAcquire(2) assertAcquire(3) - cleanup() + sb.Destroy() - sb, cleanup, err = sectorbuilder.TempSectorbuilder(sectorSize, ds) + sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) if err != nil { t.Fatalf("%+v", err) } @@ -124,5 +130,8 @@ func TestAcquireID(t *testing.T) { assertAcquire(5) assertAcquire(6) - cleanup() + sb.Destroy() + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } } From 703781765985afcc117b56e5e6336e3e28f2c11a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 20:22:58 +0100 Subject: [PATCH 199/230] mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 665d7d4be..7e29dcdf1 100644 --- a/go.sum +++ b/go.sum @@ -528,8 +528,6 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f h1:+GFA37QICd1Axd2n9uzjtvPjxJJI5PU78vpvam+hI4U= -github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 h1:ct50KYdZHcdOnTAuSgppw5MZKTa3RA63FX28m0l9Aeg= github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= From 2c66e20a1e30d55a1bf5fc874dfb407add038865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 20:55:03 +0100 Subject: [PATCH 200/230] padreader: Some more testcases --- lib/padreader/padreader_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/padreader/padreader_test.go b/lib/padreader/padreader_test.go index bf8464806..551393b4e 100644 --- a/lib/padreader/padreader_test.go +++ b/lib/padreader/padreader_test.go @@ -7,6 +7,12 @@ import ( func TestComputePaddedSize(t *testing.T) { assert.Equal(t, uint64(1040384), PaddedSize(1000000)) + assert.Equal(t, uint64(1016), PaddedSize(548)) + assert.Equal(t, uint64(1016), PaddedSize(1015)) + assert.Equal(t, uint64(1016), PaddedSize(1016)) + assert.Equal(t, uint64(2032), PaddedSize(1017)) + + assert.Equal(t, uint64(2032), PaddedSize(1024)) assert.Equal(t, uint64(4064), PaddedSize(2048)) } From ca56299076e9c18a99b6ed6421d0358135a7f229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:11:36 +0100 Subject: [PATCH 201/230] circle: force cache refresh --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c1106b210..4f4acb5bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,14 +24,14 @@ commands: - restore_cache: name: Restore parameters cache keys: - - 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' - - 'v2-lotus-params-' + - 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' + - 'v15-lotus-params-' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params - save_cache: name: Save parameters cache - key: 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' + key: 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}' paths: - /var/tmp/filecoin-proof-parameters/ From 8c39486736f45df54ef9487bb945d36b265b53c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:11:56 +0100 Subject: [PATCH 202/230] some import fixes --- chain/deals/client_states.go | 5 ++--- chain/events/events_height.go | 3 ++- chain/market/fundmgr.go | 1 + cmd/lotus-storage-miner/sectors.go | 2 +- node/builder.go | 2 +- node/impl/full.go | 8 ++++---- storage/miner.go | 7 +++---- storage/post.go | 3 +-- storage/sector_states.go | 8 ++++---- 9 files changed, 19 insertions(+), 20 deletions(-) diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 26ac0ce68..678305c95 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -4,15 +4,14 @@ import ( "bytes" "context" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/lib/cborutil" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborutil" ) type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) diff --git a/chain/events/events_height.go b/chain/events/events_height.go index 3c03f7571..524efd0d5 100644 --- a/chain/events/events_height.go +++ b/chain/events/events_height.go @@ -2,9 +2,10 @@ package events import ( "context" - "go.opencensus.io/trace" "sync" + "go.opencensus.io/trace" + "github.com/filecoin-project/lotus/chain/types" ) diff --git a/chain/market/fundmgr.go b/chain/market/fundmgr.go index 05716a428..9f8068ae2 100644 --- a/chain/market/fundmgr.go +++ b/chain/market/fundmgr.go @@ -52,6 +52,7 @@ func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr address.Address, am avail = types.NewInt(0) } fm.available[addr] = avail + fm.lk.Unlock() smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{ diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index b9075e443..627e11c14 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,11 +2,11 @@ package main import ( "fmt" - "github.com/filecoin-project/lotus/api" "strconv" "gopkg.in/urfave/cli.v2" + "github.com/filecoin-project/lotus/api" lcli "github.com/filecoin-project/lotus/cli" ) diff --git a/node/builder.go b/node/builder.go index 147366db8..0956a67c1 100644 --- a/node/builder.go +++ b/node/builder.go @@ -3,7 +3,6 @@ package node import ( "context" "errors" - "github.com/filecoin-project/lotus/chain/market" "time" blockstore "github.com/ipfs/go-ipfs-blockstore" @@ -21,6 +20,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/deals" + "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/metrics" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" diff --git a/node/impl/full.go b/node/impl/full.go index 6ce7fa91e..76cdc5de4 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -2,13 +2,13 @@ package impl import ( "context" - "github.com/filecoin-project/lotus/node/impl/market" - - "github.com/filecoin-project/lotus/node/impl/client" - "github.com/filecoin-project/lotus/node/impl/paych" logging "github.com/ipfs/go-log" + "github.com/filecoin-project/lotus/node/impl/client" + "github.com/filecoin-project/lotus/node/impl/market" + "github.com/filecoin-project/lotus/node/impl/paych" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/miner" diff --git a/storage/miner.go b/storage/miner.go index c956fd115..5cc8d37ed 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -2,14 +2,11 @@ package storage import ( "context" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "sync" - "github.com/filecoin-project/lotus/lib/statestore" - "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log" "github.com/libp2p/go-libp2p-core/host" "github.com/pkg/errors" @@ -19,6 +16,8 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/lotus/lib/statestore" ) var log = logging.Logger("storageminer") diff --git a/storage/post.go b/storage/post.go index 13a635c0d..ad1050dfb 100644 --- a/storage/post.go +++ b/storage/post.go @@ -2,18 +2,17 @@ package storage import ( "context" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "time" "github.com/ipfs/go-cid" "go.opencensus.io/trace" - "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (m *Miner) beginPosting(ctx context.Context) { diff --git a/storage/sector_states.go b/storage/sector_states.go index f94b06072..d91e3cead 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -2,15 +2,15 @@ package storage import ( "context" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" "github.com/pkg/errors" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type providerHandlerFunc func(ctx context.Context, deal SectorInfo) (func(*SectorInfo), error) From aaa073bdf8312a7bbfa6c492e4e4e2113220b567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:18:16 +0100 Subject: [PATCH 203/230] deals: Inactive deal in client.sealing is an error --- chain/deals/client_states.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index 678305c95..c01d0f430 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -195,7 +195,7 @@ func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal } if sd.ActivationEpoch == 0 { - return true, nil + return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height()) } log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch) From f3ae7eb7a3bd2f68ddcf512eb24336012f31e1f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:25:02 +0100 Subject: [PATCH 204/230] events: Test 2 block tipsets with duplicated message --- chain/events/events_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/chain/events/events_test.go b/chain/events/events_test.go index 27a3edf03..4c5177e1e 100644 --- a/chain/events/events_test.go +++ b/chain/events/events_test.go @@ -46,11 +46,24 @@ func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet func makeTs(t *testing.T, h uint64, msgcid cid.Cid) *types.TipSet { a, _ := address.NewFromString("t00") + b, _ := address.NewFromString("t02") ts, err := types.NewTipSet([]*types.BlockHeader{ { Height: h, Miner: a, + Tickets: []*types.Ticket{{[]byte{byte(h % 2)}}}, + + ParentStateRoot: dummyCid, + Messages: msgcid, + ParentMessageReceipts: dummyCid, + }, + { + Height: h, + Miner: b, + + Tickets: []*types.Ticket{{[]byte{byte((h + 1) % 2)}}}, + ParentStateRoot: dummyCid, Messages: msgcid, ParentMessageReceipts: dummyCid, @@ -494,6 +507,7 @@ func TestCalled(t *testing.T) { err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) { return false, true, nil }, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) { + require.Equal(t, false, applied) applied = true appliedMsg = msg appliedTs = ts From 3d360167df46291c7e4ef5caada9452ba45fd63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:30:50 +0100 Subject: [PATCH 205/230] sectorbuilder: Call destroy in DI module --- lib/sectorbuilder/mock.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 16 +--------------- node/builder.go | 2 +- node/modules/storageminer.go | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go index 481b982b4..e9661ada1 100644 --- a/lib/sectorbuilder/mock.go +++ b/lib/sectorbuilder/mock.go @@ -46,7 +46,7 @@ func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) ( WorkerThreads: 2, Miner: addr, - }, ds, nil) + }, ds) if err != nil { return nil, err } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index bdc8d5a66..9c2b24af2 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -1,7 +1,6 @@ package sectorbuilder import ( - "context" "fmt" "io" "os" @@ -13,7 +12,6 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log" - "go.uber.org/fx" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/chain/address" @@ -77,7 +75,7 @@ type Config struct { MetadataDir string } -func New(cfg *Config, ds dtypes.MetadataDS, lc fx.Lifecycle) (*SectorBuilder, error) { +func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) { if cfg.WorkerThreads <= PoStReservedWorkers { return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads) } @@ -126,15 +124,6 @@ func New(cfg *Config, ds dtypes.MetadataDS, lc fx.Lifecycle) (*SectorBuilder, er rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers), } - if lc != nil { - lc.Append(fx.Hook{ - OnStop: func(context.Context) error { - sb.Destroy() - return nil - }, - }) - } - return sb, nil } @@ -196,9 +185,6 @@ func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Rea if err != nil { return PublicPieceInfo{}, err } - /*if writeUnpadded != pieceSize { - return PublicPieceInfo{}, xerrors.Errorf("writeUnpadded != pieceSize: %d != %d", writeUnpadded, pieceSize) - }*/ if err := stagedFile.Close(); err != nil { return PublicPieceInfo{}, err diff --git a/node/builder.go b/node/builder.go index 0956a67c1..da37bb95f 100644 --- a/node/builder.go +++ b/node/builder.go @@ -230,7 +230,7 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner }, Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), - Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), + Override(new(*sectorblocks.SectorBlocks), modules.SectorBuilder), Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 3c827c550..0c97710cb 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -176,6 +176,22 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro return nil } +func SectorBuilder(lc fx.Lifecycle, cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) { + sb, err := sectorbuilder.New(cfg, ds) + if err != nil { + return nil, err + } + + lc.Append(fx.Hook{ + OnStop: func(context.Context) error { + sb.Destroy() + return nil + }, + }) + + return sb, nil +} + func SealTicketGen(api api.FullNode) storage.TicketFn { return func(ctx context.Context) (*sectorbuilder.SealTicket, error) { ts, err := api.ChainHead(ctx) From d87d8a6db64a9f28a2b84dacffebe11d5e2cf28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:31:48 +0100 Subject: [PATCH 206/230] peermgr: Undisable bootstrap --- peermgr/peermgr.go | 1 - 1 file changed, 1 deletion(-) diff --git a/peermgr/peermgr.go b/peermgr/peermgr.go index 3c4053da1..4ad64a5a1 100644 --- a/peermgr/peermgr.go +++ b/peermgr/peermgr.go @@ -44,7 +44,6 @@ type PeerMgr struct { } func NewPeerMgr(h host.Host, dht *dht.IpfsDHT, bootstrap dtypes.BootstrapPeers) *PeerMgr { - bootstrap = nil pm := &PeerMgr{ h: h, dht: dht, From f89f6e50e33cb2598194e1a6c53c6f56b080e768 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Fri, 8 Nov 2019 21:39:40 +0100 Subject: [PATCH 207/230] use bigger circleci machines --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdf73642..73ed4dd7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ executors: golang: docker: - image: circleci/golang:1.13 + resource_class: 2xlarge commands: install-deps: @@ -155,7 +156,7 @@ jobs: workflows: - version: 2 + version: 2.1 ci: jobs: - lint-changes: From 44a512aa20ce5cce1ef601f0b58964b4343777a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 21:46:47 +0100 Subject: [PATCH 208/230] Update go-sectorbuilder with gpu fixes --- extern/go-sectorbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 181742fdd..e198d9050 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 181742fdd08f74f1c9de54fca0f5658706039bc0 +Subproject commit e198d9050b5dde6c19bef3593506597afe00b6cb From fa67ccb5f35129c7431470722a85ddbd8adca5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 8 Nov 2019 22:00:01 +0100 Subject: [PATCH 209/230] Fix storageminer sectorblocks constructor --- node/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index da37bb95f..183472dbc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -229,8 +229,8 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner }, - Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New), - Override(new(*sectorblocks.SectorBlocks), modules.SectorBuilder), + Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder), + Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(storage.TicketFn), modules.SealTicketGen), Override(new(*storage.Miner), modules.StorageMiner), From aaacfd15cd635c828f1b31adffdd6fc9977577bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 00:06:07 +0100 Subject: [PATCH 210/230] Add comments to fillersFromRem --- node/impl/storminer.go | 2 +- storage/garbage.go | 8 +++++--- storage/sealing_utils.go | 25 ++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 49c807412..f339fd66a 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -35,7 +35,7 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error { - return sm.Miner.StoreGarbageData(ctx) + return sm.Miner.StoreGarbageData() } func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) { diff --git a/storage/garbage.go b/storage/garbage.go index 28e33e77d..f61ff50cc 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -62,7 +62,6 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPiece return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr) } - // TODO: We may want this to happen after fetching data smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ To: actors.StorageMarketAddress, From: m.worker, @@ -112,9 +111,12 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPiece return out, nil } -func (m *Miner) StoreGarbageData(_ context.Context) error { - ctx := context.TODO() +func (m *Miner) StoreGarbageData() error { go func() { + ctx := context.TODO() // we can't use the context from command which invokes + // this, as we run everything here async, and it's cancelled when the + // command exits + size := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize()) sid, err := m.sb.AcquireSectorId() diff --git a/storage/sealing_utils.go b/storage/sealing_utils.go index 26591e0ac..d8d5225bb 100644 --- a/storage/sealing_utils.go +++ b/storage/sealing_utils.go @@ -7,13 +7,36 @@ import ( ) func fillersFromRem(toFill uint64) ([]uint64, error) { - toFill += toFill / 127 // convert to in-sector bytes for easier math + // Convert to in-sector bytes for easier math: + // + // Sector size to user bytes ratio is constant, e.g. for 1024B we have 1016B + // of user-usable data. + // + // (1024/1016 = 128/127) + // + // Given that we can get sector size by simply adding 1/127 of the user + // bytes + // + // (we convert to sector bytes as they are nice round binary numbers) + toFill += toFill / 127 + + // We need to fill the sector with pieces that are powers of 2. Conveniently + // computers store numbers in binary, which means we can look at 1s to get + // all the piece sizes we need to fill the sector. It also means that number + // of pieces is the number of 1s in the number of remaining bytes to fill out := make([]uint64, bits.OnesCount64(toFill)) for i := range out { + // Extract the next lowest non-zero bit next := bits.TrailingZeros64(toFill) psize := uint64(1) << next + // e.g: if the number is 0b010100, psize will be 0b000100 + + // set that bit to 0 by XORing it, so the next iteration looks at the + // next bit toFill ^= psize + + // Add the piece size to the list of pieces we need to create out[i] = sectorbuilder.UserBytesForSectorSize(psize) } return out, nil From 5de600909efede3d4c59a0db36493609701d4e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 00:15:25 +0100 Subject: [PATCH 211/230] Make preparePost more readable Co-Authored-By: Whyrusleeping --- storage/post.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/post.go b/storage/post.go index ad1050dfb..468a010a5 100644 --- a/storage/post.go +++ b/storage/post.go @@ -149,7 +149,9 @@ func (p *post) preparePost(ctx context.Context) error { } p.sset = sset - r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, int(int64(p.ts.Height())-int64(p.ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math + // Compute how many blocks back we have to look from the given tipset for the PoSt challenge + challengeLookback := int((int64(p.ts.Height())-int64(p.ppe))+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback)) + r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, challengeLookback) if err != nil { return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err) } From ed64e52e3a5919cc6b3ed6bf982292eff8402ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 00:20:42 +0100 Subject: [PATCH 212/230] sectorbuilder: apply some review suggestions --- cmd/lotus-storage-miner/info.go | 2 +- lib/sectorbuilder/sectorbuilder.go | 10 +++++----- storage/post.go | 4 +++- storage/sealing.go | 5 +++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 54184a63c..074540faf 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -78,7 +78,7 @@ var Units = []string{"B", "KiB", "MiB", "GiB", "TiB"} func sizeStr(size uint64) string { i := 0 unitSize := float64(size) - for unitSize >= 1024 && i < len(Units) - 1 { + for unitSize >= 1024 && i < len(Units)-1 { unitSize = unitSize / 1024 i++ } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 9c2b24af2..35f37ac2d 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -49,10 +49,10 @@ const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { handle unsafe.Pointer - ds dtypes.MetadataDS - idLk sync.Mutex + ds dtypes.MetadataDS + idLk sync.Mutex - ssize uint64 + ssize uint64 Miner address.Address @@ -112,9 +112,9 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) { sb := &SectorBuilder{ handle: sbp, - ds: ds, + ds: ds, - ssize: cfg.SectorSize, + ssize: cfg.SectorSize, stagedDir: cfg.StagedDir, sealedDir: cfg.SealedDir, diff --git a/storage/post.go b/storage/post.go index ad1050dfb..3d6ac56b3 100644 --- a/storage/post.go +++ b/storage/post.go @@ -149,7 +149,9 @@ func (p *post) preparePost(ctx context.Context) error { } p.sset = sset - r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, int(int64(p.ts.Height())-int64(p.ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math + // Compute how many blocks back we have to look from the given tipset for the PoSt challenge + challengeLookback := int((int64(p.ts.Height()) - int64(p.ppe)) + int64(build.PoStChallangeTime) + int64(build.PoStRandomnessLookback)) + r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, challengeLookback) if err != nil { return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err) } diff --git a/storage/sealing.go b/storage/sealing.go index 232ddf5c4..bffb878b8 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -205,12 +205,14 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { m.handle(ctx, sector, m.preCommitted, api.SectorNoUpdate) case api.Committing: m.handle(ctx, sector, m.committing, api.Proving) + case api.SectorNoUpdate: // noop + default: + log.Error("unexpected sector update state: %d", update.newState) } } func (m *Miner) failSector(id uint64, err error) { log.Errorf("sector %d error: %+v", id, err) - panic(err) // todo: better error handling strategy } func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Reader, dealID uint64) (uint64, error) { @@ -231,7 +233,6 @@ func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Rea func (m *Miner) newSector(ctx context.Context, sid uint64, dealID uint64, ref string, ppi sectorbuilder.PublicPieceInfo) error { si := &SectorInfo{ - State: api.UndefinedSectorState, SectorID: sid, Pieces: []Piece{ From 47b54206157935af70304ec79f35405ef70c2182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 00:26:53 +0100 Subject: [PATCH 213/230] Revert BlockDelay to 10s --- build/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params.go b/build/params.go index 2fb892379..41d832b5c 100644 --- a/build/params.go +++ b/build/params.go @@ -37,7 +37,7 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours // Consensus / Network // Seconds -const BlockDelay = 2 +const BlockDelay = 10 // Seconds const AllowableClockDrift = BlockDelay * 2 From 3b6c07901419826bf7cc0967c384e2afc416c05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 01:18:15 +0100 Subject: [PATCH 214/230] Some final parameter setting for devent 7 --- build/params.go | 14 +++++++------- lotuspond/front/src/Address.js | 2 +- storage/sealing.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build/params.go b/build/params.go index 39aadaaa1..75f49e74d 100644 --- a/build/params.go +++ b/build/params.go @@ -46,16 +46,21 @@ const AllowableClockDrift = BlockDelay * 2 const ForkLengthThreshold = 100 // Blocks (e) -const BlocksPerEpoch = 3 +const BlocksPerEpoch = 5 // Blocks const Finality = 500 +// constants for Weight calculation +// The ratio of weight contributed by short-term vs long-term factors in a given round +const WRatioNum = int64(1) +const WRatioDen = 2 + // ///// // Proofs // Blocks -const ProvingPeriodDuration = 300 +const ProvingPeriodDuration = 160 // PoStChallangeTime sets the window in which post computation should happen // Blocks @@ -103,11 +108,6 @@ const FilecoinPrecision = 1_000_000_000_000_000_000 // Blocks const HalvingPeriodBlocks = 6 * 365 * 24 * 60 * 2 -// constants for Weight calculation -// The ratio of weight contributed by short-term vs long-term factors in a given round -const WRatioNum = int64(1) -const WRatioDen = 2 - // TODO: Move other important consts here func init() { diff --git a/lotuspond/front/src/Address.js b/lotuspond/front/src/Address.js index db6870e9a..1ac98d7f2 100644 --- a/lotuspond/front/src/Address.js +++ b/lotuspond/front/src/Address.js @@ -133,7 +133,7 @@ class Address extends React.Component { let transfer = if(this.props.transfer) { - transfer =  {this.props.transfer}FIL + transfer =  {this.props.transfer}FIL } let minerInfo = diff --git a/storage/sealing.go b/storage/sealing.go index bffb878b8..77ce1312c 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -207,7 +207,7 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { m.handle(ctx, sector, m.committing, api.Proving) case api.SectorNoUpdate: // noop default: - log.Error("unexpected sector update state: %d", update.newState) + log.Errorf("unexpected sector update state: %d", update.newState) } } From 814e01cc05dd52fa0cafd56d152d1fff97c2a833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 01:18:32 +0100 Subject: [PATCH 215/230] gofmt --- chain/store/weight.go | 6 +++--- chain/types/blockheader.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/chain/store/weight.go b/chain/store/weight.go index 999166f89..53c202845 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -38,16 +38,16 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn if tpow.GreaterThan(zero) { log2P = int64(tpow.BitLen() - 1) } else { - // Not really expect to be here ... + // Not really expect to be here ... return types.EmptyInt, xerrors.Errorf("All power in the net is gone. You network might be disconnected, or the net is dead!") } - out.Add(out, big.NewInt(log2P << 8)) + out.Add(out, big.NewInt(log2P<<8)) // (wFunction(totalPowerAtTipset(ts)) * len(ts.blocks) * wRatio_num * 2^8) / (e * wRatio_den) eWeight := big.NewInt((log2P * int64(len(ts.Blocks())) * build.WRatioNum) << 8) - eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch * build.WRatioDen))) + eWeight.Div(eWeight, big.NewInt(int64(build.BlocksPerEpoch*build.WRatioDen))) out.Add(out, eWeight) return types.BigInt{Int: out}, nil diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 204208e0f..d65a068d8 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -170,7 +170,7 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { max(h) == 2^256-1 which in terms of integer math means: (h(vrfout) + 1) * totalPower <= e * minerPower * 2^256 - in 2^256 space, it is equivalent to: + in 2^256 space, it is equivalent to: h(vrfout) * totalPower < e * minerPower * 2^256 */ @@ -183,7 +183,7 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool { // rhs = minerPower << 256 rhs := new(big.Int).Lsh(mpow.Int, 256) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) - + // h(vrfout) * totalPower < e * minerPower * 2^256? return lhs.Cmp(rhs) == -1 } From 18bcc9df8accb6230343b3e215bcc15ae9590ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 09:26:31 +0100 Subject: [PATCH 216/230] Update install instructions --- README.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e40dd3038..2a4cb394c 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,35 @@ In order to run lotus, please do the following: - bzr (some go dependency needs this) - jq - pkg-config -- opencl-headers -- opencl-driver - opencl-icd-loader +- opencl driver (like nvidia-opencl on arch) (for GPU acceleration) +- opencl-headers (build) +- rustup (proofs build) +- llvm (proofs build) +- clang (proofs build) +Arch (run): +```sh +sudo pacman -Syu opencl-icd-loader +``` + +Arch (build): +```sh +sudo pacman -Syu go gcc git bzr jq pkg-config opencl-icd-loader opencl-headers +``` + +Ubuntu / Debian (run): +```sh +sudo apt update +sudo apt install mesa-opencl-icd ocl-icd-opencl-dev +``` + +Ubuntu (build): +```sh +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install golang-go gcc git bzr jq pkg-config mesa-opencl-icd ocl-icd-opencl-dev +``` 2. Clone this repo & `cd` into it ``` From 0734e5329de7e6905cc405c3b719094fcd973353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 09:26:42 +0100 Subject: [PATCH 217/230] Devnet 7 --- build/bootstrap/bootstrappers.pi | 2 +- build/bootstrap/root.pi | 2 +- build/genesis/devnet.car | Bin 2218 -> 2213 bytes 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi index 0319a6d97..4d803d299 100644 --- a/build/bootstrap/bootstrappers.pi +++ b/build/bootstrap/bootstrappers.pi @@ -1 +1 @@ -/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWPTYfTi7B1mQpbyRaCT65Trf7zW9npmihcF1xPfZogrzh +/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWLM9S94b9mnHD2fbWvQzXudoapHjbBbFA1T4QVft65Kym diff --git a/build/bootstrap/root.pi b/build/bootstrap/root.pi index b03183ddf..80ed9013a 100644 --- a/build/bootstrap/root.pi +++ b/build/bootstrap/root.pi @@ -1 +1 @@ -/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWEwGkxuG4BSvWUU8Sf69rxUYjWHAMT2qFVBhzGJZAXc6v +/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWB4pVpuRAcW2873V6m3uPWevKcZ1VWr1FGPUTwDZUN5mg diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index ea19dc0d723bf81901302a36094dfa94e20d1c75..7d400c1110a6e071f897581cfd88752e3e0ed50d 100644 GIT binary patch delta 752 zcmZ1_xKwb0NMKp`>AB_)_?!Oz-9KSwkMbGaDb|&oNzeSxD1MwH@%BJ-dRb~wab|uV zV>c6H;esbj3WPLFO!ZKhXQXwjA^ZT#ZayBz!ud+pmO@T0A!k0cIlhhLk4VuxKe?XO zY%(*Wf&H=az+k4Noc!c$h2qTgyu^~yqEsOGzyh(0pgBd19tzgxpN`EBy(!LQ&K%LM zzJ3;e^4s)Bx9)$N?#1zY-dZt($+sBKD1@(`VN|l*Gh~<3-yH%szC8}z9Vua*xgun) zSE8%pq|1t%k1>fc))O+S>4sK>Is?RQ>I}K!XIXCDkC>VLKGI{7{F`(2I!~J`E?#^s zd_?*6^E8RH>5RmwYzlW^00VCb1tJx|%%+8jhK)@XhRls@YuA|{T@ig{^AhWt(xq=D z`q(?(F8notHLf&a$&DgL)=4H0Di72o{$ID*Lc7*iMj~Z$0gLS9<19&9+n}x>6aw7~ zfUy!bxt@btC;LKZ^Li)ARN<;69-bc8nN7+!r{$hDHn{2iUgl6r+6zYp5Oih`?r?CJ sT+gOoKL=_DA&o5&qka0m^hh#sB~S delta 817 zcmZ1~xJqz>NZ?|hbL$k3E446QSoO%_^ZT8DPETBY*ev6F@!$N5ss4(2>1C-!#hLke zjNMF(g$tfADG<^yG1WuiBin1!>iS9RHzeQxf33z#adG*{Pd?49s^&BGCvJIqdctIX zRj0W~Q%rt|Ul5+BsvlWUn)AJHbN{do~-~$W9E`sJ1F?uL8Oq%Yj$*|ChC%r61 zwVvt49v%mU{@FG5JDMzS-n_07Klv8p83oP!onKR86?a6QHJta+ciF7u;xZrkzTD4B z_oW34gxJn*KE@=*SWn2PrW;xj>I@LK$y`44zg+3;1inl5-S+2R9Z|?Sr|hwwdD^lR zCKIbPrFMz5*^I=gY>JR_U|;~EdIl_@XnGDKCp_)>B2 zlSC?5vciygnsslbqik`J>hj4wUzm4ne&gu7rXaPK?~jlA%`k_KrMD-pTiG6_n{%=1 z_}gzQ^Y2Nd$T5a8@TO(v=o{)MCMV~Y=9Q#PzQ*DowjJtfLLt_J6c+mgUh^8QKQc{s zYs#kBt_<;09V*x6YcAQ>zt6Aqf5_Co4i3N=536@%06}L4;SL7}hB;7s27X?EpMqo7PPOfKFV{c|)Xm)65*u0YM4 Date: Sat, 9 Nov 2019 09:46:51 +0100 Subject: [PATCH 218/230] Readme: some minor updates --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2a4cb394c..68a78cfa0 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ $ cd lotus/ 3. Build and install the source code ``` -$ make +$ make clean all $ sudo make install ``` @@ -89,16 +89,12 @@ $ lotus net peers | wc -l 2 # number of peers ``` -[wait for the chain to finish syncing] - -You can follow sync status with: +Wait for the chain to finish syncing: ```sh -$ watch lotus sync status +$ lotus sync wait ``` -then view latest block height along with other network metrics at the https://lotus-metrics.kittyhawk.wtf/chain. - -[It may take a few minutes for the chain to finish syncing. You will see `Height: 0` until the full chain is synced and validated.] +You can view latest block height along with other network metrics at the https://lotus-metrics.kittyhawk.wtf/chain. ### Basics From 59fbc976d078df5f60c454b8ff408a2667286bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 10:02:55 +0100 Subject: [PATCH 219/230] storageminer: Missing new line in info cmd --- cmd/lotus-storage-miner/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 074540faf..45e4d0e9a 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -57,7 +57,7 @@ var infoCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("Worker use: %d / %d (+%d)", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved) + fmt.Printf("Worker use: %d / %d (+%d)\n", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved) sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { From 3f4d454f6eb8fd4dd9054cbf19c8218f12ea3745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 10:49:43 +0100 Subject: [PATCH 220/230] storageminer: Print proving period in info cmd --- build/paramfetch.go | 1 + cmd/lotus-storage-miner/info.go | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/build/paramfetch.go b/build/paramfetch.go index 55dfd62bc..81b9be55d 100644 --- a/build/paramfetch.go +++ b/build/paramfetch.go @@ -110,6 +110,7 @@ func (ft *fetch) checkFile(path string, info paramFile) error { sum := h.Sum(nil) strSum := hex.EncodeToString(sum[:16]) if strSum == info.Digest { + log.Infof("Parameter file %s is ok", path) return nil } diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 45e4d0e9a..864050170 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -7,6 +7,7 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -59,12 +60,24 @@ var infoCmd = &cli.Command{ } fmt.Printf("Worker use: %d / %d (+%d)\n", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved) + ppe, err := api.StateMinerProvingPeriodEnd(ctx, maddr, nil) + if err != nil { + return err + } + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + pdiff := ppe-head.Height() + pdifft := pdiff * build.BlockDelay + fmt.Printf("Proving Period: %d, in %d Blocks (~%dm %ds)\n", ppe, pdiff, pdifft / 60, pdifft % 60) + sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { return err } - fmt.Println(sinfo) + fmt.Println("Sectors: ", sinfo) // TODO: grab actr state / info // * Sealed sectors (count / bytes) From 81bd0bc5178ab9572686681274207bb463f84425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 11:42:36 +0100 Subject: [PATCH 221/230] Test to reproduce post error after restart --- cmd/lotus-storage-miner/info.go | 2 +- lib/sectorbuilder/sectorbuilder_test.go | 35 ++++++++++++++++++++++++- storage/post.go | 11 ++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 864050170..aed867c1e 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -68,7 +68,7 @@ var infoCmd = &cli.Command{ if err != nil { return err } - pdiff := ppe-head.Height() + pdiff := int64(ppe-head.Height()) pdifft := pdiff * build.BlockDelay fmt.Printf("Proving Period: %d, in %d Blocks (~%dm %ds)\n", ppe, pdiff, pdifft / 60, pdifft % 60) diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index e02368913..bb965c2bb 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -27,10 +27,23 @@ func TestSealAndVerify(t *testing.T) { t.Fatalf("%+v", err) } - sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, datastore.NewMapDatastore()) + ds := datastore.NewMapDatastore() + + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + t.Fatal(err) + } + + sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) if err != nil { t.Fatalf("%+v", err) } + cleanup := func() { + sb.Destroy() + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + } defer cleanup() dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) @@ -94,6 +107,26 @@ func TestSealAndVerify(t *testing.T) { if !ok { t.Fatal("bad post") } + + // Restart sectorbuilder, re-run post + sb.Destroy() + sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) + if err != nil { + t.Fatalf("%+v", err) + } + + postProof, err = sb.GeneratePoSt(ssi, cSeed, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + + ok, err = sectorbuilder.VerifyPost(sb.SectorSize(), ssi, cSeed, postProof, []uint64{}) + if err != nil { + t.Fatalf("%+v", err) + } + if !ok { + t.Fatal("bad post") + } } func TestAcquireID(t *testing.T) { diff --git a/storage/post.go b/storage/post.go index 3d6ac56b3..0bcd66bb5 100644 --- a/storage/post.go +++ b/storage/post.go @@ -147,6 +147,10 @@ func (p *post) preparePost(ctx context.Context) error { if err != nil { return xerrors.Errorf("failed to get proving set for miner: %w", err) } + if len(sset) == 0 { + log.Warn("empty proving set! (ts.H: %d)", p.ts.Height()) + } + p.sset = sset // Compute how many blocks back we have to look from the given tipset for the PoSt challenge @@ -181,7 +185,7 @@ func (p *post) runPost(ctx context.Context) error { log.Infow("running PoSt", "delayed-by", int64(p.ts.Height())-(int64(p.ppe)-int64(build.PoStChallangeTime)), - "chain-random", p.r, "ppe", p.ppe, "height", p.ts.Height()) + "chain-random", p.r, "ppe", p.ppe, "height", p.ts.Height(), "sectors", len(p.sset)) tsStart := time.Now() var faults []uint64 // TODO @@ -189,6 +193,9 @@ func (p *post) runPost(ctx context.Context) error { var seed [32]byte copy(seed[:], p.r) + vals := p.sortedSectorInfo() + log.Infof("SSI: %+v", vals.Values()) + proof, err := p.m.sb.GeneratePoSt(p.sortedSectorInfo(), seed, faults) if err != nil { return xerrors.Errorf("running post failed: %w", err) @@ -270,7 +277,7 @@ func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipS ts: ts, }).doPost(ctx) if err != nil { - return err + return xerrors.Errorf("doPost: %w", err) } m.scheduleNextPost(ppe + build.ProvingPeriodDuration) From 4e6e83149a173e005b670d950097a19b08df5e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 12:26:47 +0100 Subject: [PATCH 222/230] Update sectorbuilder with a fix --- extern/go-sectorbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index e198d9050..7c892757e 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit e198d9050b5dde6c19bef3593506597afe00b6cb +Subproject commit 7c892757e0465fbdb38c342c4ea890775f63a59a From c18127bf000db83bade209a6054de2d8c6168625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 18:51:04 +0100 Subject: [PATCH 223/230] townhall: More colors, weight/height diff --- cmd/lotus-townhall/townhall/src/App.js | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-townhall/townhall/src/App.js b/cmd/lotus-townhall/townhall/src/App.js index 058e9e32b..aa6c78996 100644 --- a/cmd/lotus-townhall/townhall/src/App.js +++ b/cmd/lotus-townhall/townhall/src/App.js @@ -1,6 +1,15 @@ import React from 'react'; import './App.css'; +function colForH(besth, height) { + const diff = besth - height + if(diff === 0) return '#6f6' + if(diff === 1) return '#df4' + if(diff < 4) return '#ff0' + if(diff < 10) return '#f60' + return '#f00' +} + class App extends React.Component { constructor(props) { super(props); @@ -21,18 +30,18 @@ class App extends React.Component { } render() { - let best = Object.keys(this.state).map(k => this.state[k]).reduce((p, n) => p > n.Height ? p : n.Height, -1) - console.log(best) + let besth = Object.keys(this.state).map(k => this.state[k]).reduce((p, n) => p > n.Height ? p : n.Height, -1) + let bestw = Object.keys(this.state).map(k => this.state[k]).reduce((p, n) => p > n.Weight ? p : n.Weight, -1) return {Object.keys(this.state).map(k => [k, this.state[k]]).map(([k, v]) => { let mnrs = v.Blocks.map(b =>  m:{b.Miner}) - let l = [, , , ] - if (best !== v.Height) { - l = {l} - } else { + let l = [, + , + , + , + ] l = {l} - } return l }) From bda6d7e1191efdf659a36531e6ed5da0e09f339c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 9 Nov 2019 12:14:40 -0800 Subject: [PATCH 224/230] Don't try syncing to a chain if its not clearly heavier --- chain/blocksync.go | 52 ++++++++++++++++++++++++++++++-------------- chain/store/store.go | 11 +++++++--- chain/sync.go | 11 ++++++++++ go.sum | 5 +++++ storage/post.go | 2 ++ 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/chain/blocksync.go b/chain/blocksync.go index 71140f2e0..57a4890f3 100644 --- a/chain/blocksync.go +++ b/chain/blocksync.go @@ -130,7 +130,7 @@ func (bss *BlockSyncService) processRequest(ctx context.Context, req *BlockSyncR chain, err := bss.collectChainSegment(req.Start, req.RequestLength, opts) if err != nil { - log.Error("encountered error while responding to block sync request: ", err) + log.Warn("encountered error while responding to block sync request: ", err) return &BlockSyncResponse{ Status: 203, }, nil @@ -149,7 +149,7 @@ func (bss *BlockSyncService) collectChainSegment(start []cid.Cid, length uint64, var bst BSTipSet ts, err := bss.cs.LoadTipSet(cur) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed loading tipset %s: %w", cur, err) } if opts.IncludeMessages { @@ -222,8 +222,8 @@ func (bss *BlockSyncService) gatherMessages(ts *types.TipSet) ([]*types.Message, } type BlockSync struct { - bserv bserv.BlockService - newStream NewStreamFunc + bserv bserv.BlockService + host host.Host syncPeersLk sync.Mutex syncPeers map[peer.ID]struct{} @@ -232,21 +232,11 @@ type BlockSync struct { func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host) *BlockSync { return &BlockSync{ bserv: bserv, - newStream: h.NewStream, + host: h, syncPeers: make(map[peer.ID]struct{}), } } -func (bs *BlockSync) getPeers() []peer.ID { - bs.syncPeersLk.Lock() - defer bs.syncPeersLk.Unlock() - var out []peer.ID - for p := range bs.syncPeers { - out = append(out, p) - } - return out -} - func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error { switch res.Status { case 101: // Partial Response @@ -396,8 +386,18 @@ func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) { } func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) { - s, err := bs.newStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID) + ctx, span := trace.StartSpan(ctx, "sendRequestToPeer") + defer span.End() + + if span.IsRecordingEvents() { + span.AddAttributes( + trace.StringAttribute("peer", p.Pretty()), + ) + } + + s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID) if err != nil { + bs.RemovePeer(p) return nil, err } @@ -452,6 +452,26 @@ func (bs *BlockSync) AddPeer(p peer.ID) { bs.syncPeers[p] = struct{}{} } +func (bs *BlockSync) RemovePeer(p peer.ID) { + bs.syncPeersLk.Lock() + defer bs.syncPeersLk.Unlock() + delete(bs.syncPeers, p) +} + +func (bs *BlockSync) getPeers() []peer.ID { + bs.syncPeersLk.Lock() + defer bs.syncPeersLk.Unlock() + var out []peer.ID + for p := range bs.syncPeers { + out = append(out, p) + } + return out +} + +func (bs *BlockSync) logPeerQuality(p peer.ID) { + +} + func (bs *BlockSync) FetchMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.Message, error) { out := make([]*types.Message, len(cids)) diff --git a/chain/store/store.go b/chain/store/store.go index 3bbdaa0f7..dcef64027 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -231,7 +231,7 @@ func (cs *ChainStore) MaybeTakeHeavierTipSet(ctx context.Context, ts *types.TipS // TODO: don't do this for initial sync. Now that we don't have a // difference between 'bootstrap sync' and 'caught up' sync, we need // some other heuristic. - return cs.takeHeaviestTipSet(ts) + return cs.takeHeaviestTipSet(ctx, ts) } return nil } @@ -267,7 +267,10 @@ func (cs *ChainStore) reorgWorker(ctx context.Context) chan<- reorg { return out } -func (cs *ChainStore) takeHeaviestTipSet(ts *types.TipSet) error { +func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) error { + ctx, span := trace.StartSpan(ctx, "takeHeaviestTipSet") + defer span.End() + if cs.heaviest != nil { // buf if len(cs.reorgCh) > 0 { log.Warnf("Reorg channel running behind, %d reorgs buffered", len(cs.reorgCh)) @@ -280,6 +283,8 @@ func (cs *ChainStore) takeHeaviestTipSet(ts *types.TipSet) error { log.Warnf("no heaviest tipset found, using %s", ts.Cids()) } + span.AddAttributes(trace.BoolAttribute("newHead", true)) + log.Debugf("New heaviest tipset! %s", ts.Cids()) cs.heaviest = ts @@ -296,7 +301,7 @@ func (cs *ChainStore) takeHeaviestTipSet(ts *types.TipSet) error { func (cs *ChainStore) SetHead(ts *types.TipSet) error { cs.heaviestLk.Lock() defer cs.heaviestLk.Unlock() - return cs.takeHeaviestTipSet(ts) + return cs.takeHeaviestTipSet(context.TODO(), ts) } func (cs *ChainStore) Contains(ts *types.TipSet) (bool, error) { diff --git a/chain/sync.go b/chain/sync.go index 80df33d27..46073ea2a 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -111,11 +111,18 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { return } + syncer.peerHeadsLk.Lock() syncer.peerHeads[from] = fts.TipSet() syncer.peerHeadsLk.Unlock() syncer.Bsync.AddPeer(from) + bestPweight := syncer.store.GetHeaviestTipSet().Blocks()[0].ParentWeight + if fts.TipSet().Blocks()[0].ParentWeight.LessThan(bestPweight) { + log.Warn("incoming tipset does not appear to be better than our best chain, ignoring for now") + return + } + go func() { if err := syncer.Sync(ctx, fts.TipSet()); err != nil { log.Errorf("sync error: %+v", err) @@ -724,6 +731,8 @@ loop: log.Errorf("failed to get blocks: %+v", err) + span.AddAttributes(trace.StringAttribute("error", err.Error())) + // This error will only be logged above, return nil, xerrors.Errorf("failed to get blocks: %w", err) } @@ -906,6 +915,8 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error return err } + span.AddAttributes(trace.Int64Attribute("syncChainLength", int64(len(headers)))) + if !headers[0].Equals(ts) { log.Errorf("collectChain headers[0] should be equal to sync target. Its not: %s != %s", headers[0].Cids(), ts.Cids()) } diff --git a/go.sum b/go.sum index 7e29dcdf1..e3f5c0e86 100644 --- a/go.sum +++ b/go.sum @@ -22,7 +22,9 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -474,6 +476,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -481,6 +484,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -683,6 +687,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/storage/post.go b/storage/post.go index 3d6ac56b3..bdc0a434d 100644 --- a/storage/post.go +++ b/storage/post.go @@ -142,6 +142,7 @@ func (p *post) doPost(ctx context.Context) error { func (p *post) preparePost(ctx context.Context) error { ctx, span := trace.StartSpan(ctx, "storage.preparePost") defer span.End() + log.Info("preparePost") sset, err := p.m.api.StateMinerProvingSet(ctx, p.m.maddr, p.ts) if err != nil { @@ -251,6 +252,7 @@ func (p *post) waitCommit(ctx context.Context) error { log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode) // TODO: Do something } + log.Infof("Post made it on chain! (height=%d)", rec.TipSet.Height()) return nil } From 904fdad4c651b4abddf43e7934f4eb70b832f2e0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 9 Nov 2019 15:00:22 -0800 Subject: [PATCH 225/230] extract blocksync into its own package --- chain/{ => blocksync}/blocksync.go | 5 +- chain/{ => blocksync}/cbor_gen.go | 122 +++++++++++++++-------------- chain/sync.go | 7 +- chain/types/cbor_gen.go | 2 +- gen/main.go | 21 +++-- go.mod | 1 + go.sum | 2 + node/builder.go | 5 +- node/hello/hello.go | 4 +- node/modules/services.go | 5 +- 10 files changed, 95 insertions(+), 79 deletions(-) rename chain/{ => blocksync}/blocksync.go (99%) rename chain/{ => blocksync}/cbor_gen.go (78%) diff --git a/chain/blocksync.go b/chain/blocksync/blocksync.go similarity index 99% rename from chain/blocksync.go rename to chain/blocksync/blocksync.go index 57a4890f3..3820e3688 100644 --- a/chain/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -1,4 +1,4 @@ -package chain +package blocksync import ( "bufio" @@ -21,10 +21,13 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" + logging "github.com/ipfs/go-log" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" ) +var log = logging.Logger("blocksync") + type NewStreamFunc func(context.Context, peer.ID, ...protocol.ID) (inet.Stream, error) const BlockSyncProtocolID = "/fil/sync/blk/0.0.1" diff --git a/chain/cbor_gen.go b/chain/blocksync/cbor_gen.go similarity index 78% rename from chain/cbor_gen.go rename to chain/blocksync/cbor_gen.go index 30e62071b..4af61a490 100644 --- a/chain/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -1,11 +1,11 @@ -package chain +package blocksync import ( "fmt" "io" "github.com/filecoin-project/lotus/chain/types" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -15,11 +15,15 @@ import ( var _ = xerrors.Errorf func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } if _, err := w.Write([]byte{131}); err != nil { return err } - // t.t.Start ([]cid.Cid) + // t.t.Start ([]cid.Cid) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Start)))); err != nil { return err } @@ -29,19 +33,20 @@ func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error { } } - // t.t.RequestLength (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.RequestLength)); err != nil { + // t.t.RequestLength (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RequestLength))); err != nil { return err } - // t.t.Options (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Options)); err != nil { + // t.t.Options (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Options))); err != nil { return err } return nil } -func (t *BlockSyncRequest) UnmarshalCBOR(br io.Reader) error { +func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) if err != nil { @@ -55,14 +60,14 @@ func (t *BlockSyncRequest) UnmarshalCBOR(br io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Start ([]cid.Cid) + // t.t.Start ([]cid.Cid) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.Start: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -80,7 +85,7 @@ func (t *BlockSyncRequest) UnmarshalCBOR(br io.Reader) error { t.Start[i] = c } - // t.t.RequestLength (uint64) + // t.t.RequestLength (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -89,8 +94,8 @@ func (t *BlockSyncRequest) UnmarshalCBOR(br io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.RequestLength = extra - // t.t.Options (uint64) + t.RequestLength = uint64(extra) + // t.t.Options (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -99,16 +104,20 @@ func (t *BlockSyncRequest) UnmarshalCBOR(br io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Options = extra + t.Options = uint64(extra) return nil } func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } if _, err := w.Write([]byte{131}); err != nil { return err } - // t.t.Chain ([]*chain.BSTipSet) + // t.t.Chain ([]*blocksync.BSTipSet) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Chain)))); err != nil { return err } @@ -118,12 +127,12 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error { } } - // t.t.Status (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, t.Status)); err != nil { + // t.t.Status (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { return err } - // t.t.Message (string) + // t.t.Message (string) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { return err } @@ -133,7 +142,8 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error { return nil } -func (t *BlockSyncResponse) UnmarshalCBOR(br io.Reader) error { +func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) if err != nil { @@ -147,14 +157,14 @@ func (t *BlockSyncResponse) UnmarshalCBOR(br io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Chain ([]*chain.BSTipSet) + // t.t.Chain ([]*blocksync.BSTipSet) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.Chain: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -164,6 +174,7 @@ func (t *BlockSyncResponse) UnmarshalCBOR(br io.Reader) error { t.Chain = make([]*BSTipSet, extra) } for i := 0; i < int(extra); i++ { + var v BSTipSet if err := v.UnmarshalCBOR(br); err != nil { return err @@ -172,7 +183,7 @@ func (t *BlockSyncResponse) UnmarshalCBOR(br io.Reader) error { t.Chain[i] = &v } - // t.t.Status (uint64) + // t.t.Status (uint64) (uint64) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -181,39 +192,30 @@ func (t *BlockSyncResponse) UnmarshalCBOR(br io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Status = extra - // t.t.Message (string) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if maj != cbg.MajTextString { - return fmt.Errorf("expected cbor type 'text string' in input") - } - - if extra > 256*1024 { - return fmt.Errorf("string in cbor input too long") - } + t.Status = uint64(extra) + // t.t.Message (string) (string) { - buf := make([]byte, extra) - if _, err := io.ReadFull(br, buf); err != nil { + sval, err := cbg.ReadString(br) + if err != nil { return err } - t.Message = string(buf) + t.Message = string(sval) } return nil } func (t *BSTipSet) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } if _, err := w.Write([]byte{133}); err != nil { return err } - // t.t.Blocks ([]*types.BlockHeader) + // t.t.Blocks ([]*types.BlockHeader) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Blocks)))); err != nil { return err } @@ -223,7 +225,7 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.BlsMessages ([]*types.Message) + // t.t.BlsMessages ([]*types.Message) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.BlsMessages)))); err != nil { return err } @@ -233,7 +235,7 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.BlsMsgIncludes ([][]uint64) + // t.t.BlsMsgIncludes ([][]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.BlsMsgIncludes)))); err != nil { return err } @@ -248,7 +250,7 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.SecpkMessages ([]*types.SignedMessage) + // t.t.SecpkMessages ([]*types.SignedMessage) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.SecpkMessages)))); err != nil { return err } @@ -258,7 +260,7 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { } } - // t.t.SecpkMsgIncludes ([][]uint64) + // t.t.SecpkMsgIncludes ([][]uint64) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.SecpkMsgIncludes)))); err != nil { return err } @@ -275,7 +277,8 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { return nil } -func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { +func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) if err != nil { @@ -289,14 +292,14 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Blocks ([]*types.BlockHeader) + // t.t.Blocks ([]*types.BlockHeader) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.Blocks: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -306,6 +309,7 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.Blocks = make([]*types.BlockHeader, extra) } for i := 0; i < int(extra); i++ { + var v types.BlockHeader if err := v.UnmarshalCBOR(br); err != nil { return err @@ -314,14 +318,14 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.Blocks[i] = &v } - // t.t.BlsMessages ([]*types.Message) + // t.t.BlsMessages ([]*types.Message) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.BlsMessages: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -331,6 +335,7 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.BlsMessages = make([]*types.Message, extra) } for i := 0; i < int(extra); i++ { + var v types.Message if err := v.UnmarshalCBOR(br); err != nil { return err @@ -339,14 +344,14 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.BlsMessages[i] = &v } - // t.t.BlsMsgIncludes ([][]uint64) + // t.t.BlsMsgIncludes ([][]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.BlsMsgIncludes: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -366,7 +371,7 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.BlsMsgIncludes[i]: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -392,14 +397,14 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { } } - // t.t.SecpkMessages ([]*types.SignedMessage) + // t.t.SecpkMessages ([]*types.SignedMessage) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.SecpkMessages: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -409,6 +414,7 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.SecpkMessages = make([]*types.SignedMessage, extra) } for i := 0; i < int(extra); i++ { + var v types.SignedMessage if err := v.UnmarshalCBOR(br); err != nil { return err @@ -417,14 +423,14 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { t.SecpkMessages[i] = &v } - // t.t.SecpkMsgIncludes ([][]uint64) + // t.t.SecpkMsgIncludes ([][]uint64) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.SecpkMsgIncludes: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -444,7 +450,7 @@ func (t *BSTipSet) UnmarshalCBOR(br io.Reader) error { return err } if extra > 8192 { - return fmt.Errorf("array too large") + return fmt.Errorf("t.SecpkMsgIncludes[i]: array too large (%d)", extra) } if maj != cbg.MajArray { diff --git a/chain/sync.go b/chain/sync.go index 46073ea2a..e7384e02f 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" @@ -49,7 +50,7 @@ type Syncer struct { bad *BadBlockCache // handle to the block sync service - Bsync *BlockSync + Bsync *blocksync.BlockSync self peer.ID @@ -61,7 +62,7 @@ type Syncer struct { peerHeadsLk sync.Mutex } -func NewSyncer(sm *stmgr.StateManager, bsync *BlockSync, self peer.ID) (*Syncer, error) { +func NewSyncer(sm *stmgr.StateManager, bsync *blocksync.BlockSync, self peer.ID) (*Syncer, error) { gen, err := sm.ChainStore().GetGenesis() if err != nil { return nil, err @@ -882,7 +883,7 @@ func (syncer *Syncer) iterFullTipsets(ctx context.Context, headers []*types.TipS return nil } -func persistMessages(bs bstore.Blockstore, bst *BSTipSet) error { +func persistMessages(bs bstore.Blockstore, bst *blocksync.BSTipSet) error { for _, m := range bst.BlsMessages { //log.Infof("putting BLS message: %s", m.Cid()) if _, err := store.PutMessage(bs, m); err != nil { diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index fa3cddaf4..1ebe94a13 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/gen/main.go b/gen/main.go index b2b0140e7..97590eead 100644 --- a/gen/main.go +++ b/gen/main.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/paych" @@ -71,17 +72,15 @@ func main() { os.Exit(1) } - /* - err = gen.WriteTupleEncodersToFile("./chain/cbor_gen.go", "chain", - chain.BlockSyncRequest{}, - chain.BlockSyncResponse{}, - chain.BSTipSet{}, - ) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - */ + err = gen.WriteTupleEncodersToFile("./chain/blocksync/cbor_gen.go", "blocksync", + blocksync.BlockSyncRequest{}, + blocksync.BlockSyncResponse{}, + blocksync.BSTipSet{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } err = gen.WriteTupleEncodersToFile("./chain/actors/cbor_gen.go", "actors", actors.InitActorState{}, diff --git a/go.mod b/go.mod index 8e62267a7..8c16260b0 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.1.3 github.com/libp2p/go-libp2p-pnet v0.1.0 + github.com/libp2p/go-libp2p-protocol v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.0 github.com/libp2p/go-libp2p-quic-transport v0.1.1 github.com/libp2p/go-libp2p-record v0.1.1 diff --git a/go.sum b/go.sum index e3f5c0e86..545d4d988 100644 --- a/go.sum +++ b/go.sum @@ -325,6 +325,8 @@ github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+ github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA= github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE= +github.com/libp2p/go-libp2p-protocol v0.1.0 h1:HdqhEyhg0ToCaxgMhnOmUO8snQtt/kQlcjVk3UoJU3c= +github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-quic-transport v0.1.1 h1:MFMJzvsxIEDEVKzO89BnB/FgvMj9WI4GDGUW2ArDPUA= diff --git a/node/builder.go b/node/builder.go index 183472dbc..1d1e0d3e7 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/metrics" @@ -197,14 +198,14 @@ func Online() Option { // Filecoin services Override(new(*chain.Syncer), chain.NewSyncer), - Override(new(*chain.BlockSync), chain.NewBlockSyncClient), + Override(new(*blocksync.BlockSync), blocksync.NewBlockSyncClient), Override(new(*chain.MessagePool), chain.NewMessagePool), Override(new(modules.Genesis), modules.ErrorGenesis), Override(SetGenesisKey, modules.SetGenesis), Override(new(*hello.Service), hello.NewHelloService), - Override(new(*chain.BlockSyncService), chain.NewBlockSyncService), + Override(new(*blocksync.BlockSyncService), blocksync.NewBlockSyncService), Override(new(*peermgr.PeerMgr), peermgr.NewPeerMgr), Override(RunHelloKey, modules.RunHello), diff --git a/node/hello/hello.go b/node/hello/hello.go index 90af18923..880ef96f1 100644 --- a/node/hello/hello.go +++ b/node/hello/hello.go @@ -12,6 +12,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" inet "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + protocol "github.com/libp2p/go-libp2p-protocol" "github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain/store" @@ -34,8 +35,9 @@ type Message struct { GenesisHash cid.Cid } +type NewStreamFunc func(context.Context, peer.ID, ...protocol.ID) (inet.Stream, error) type Service struct { - newStream chain.NewStreamFunc + newStream NewStreamFunc cs *store.ChainStore syncer *chain.Syncer diff --git a/node/modules/services.go b/node/modules/services.go index c5c3df125..8cf298d8a 100644 --- a/node/modules/services.go +++ b/node/modules/services.go @@ -9,6 +9,7 @@ import ( "go.uber.org/fx" "github.com/filecoin-project/lotus/chain" + "github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/node/hello" @@ -37,8 +38,8 @@ func RunPeerMgr(mctx helpers.MetricsCtx, lc fx.Lifecycle, pmgr *peermgr.PeerMgr) go pmgr.Run(helpers.LifecycleCtx(mctx, lc)) } -func RunBlockSync(h host.Host, svc *chain.BlockSyncService) { - h.SetStreamHandler(chain.BlockSyncProtocolID, svc.HandleStream) +func RunBlockSync(h host.Host, svc *blocksync.BlockSyncService) { + h.SetStreamHandler(blocksync.BlockSyncProtocolID, svc.HandleStream) } func HandleIncomingBlocks(mctx helpers.MetricsCtx, lc fx.Lifecycle, pubsub *pubsub.PubSub, s *chain.Syncer) { From c74f87fd51e744fdd506fa9f22e76eb642bea108 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sun, 10 Nov 2019 11:33:08 -0800 Subject: [PATCH 226/230] split files, implement peer tracker --- chain/blocksync/blocksync.go | 316 ---------------------- chain/blocksync/blocksync_client.go | 400 ++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 4 files changed, 403 insertions(+), 316 deletions(-) create mode 100644 chain/blocksync/blocksync_client.go diff --git a/chain/blocksync/blocksync.go b/chain/blocksync/blocksync.go index 3820e3688..fd352a9ea 100644 --- a/chain/blocksync/blocksync.go +++ b/chain/blocksync/blocksync.go @@ -3,12 +3,7 @@ package blocksync import ( "bufio" "context" - "fmt" - "math/rand" - "sync" - bserv "github.com/ipfs/go-blockservice" - "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/protocol" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -16,9 +11,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/cborutil" - "github.com/filecoin-project/lotus/node/modules/dtypes" - blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" logging "github.com/ipfs/go-log" @@ -224,151 +217,6 @@ func (bss *BlockSyncService) gatherMessages(ts *types.TipSet) ([]*types.Message, return blsmsgs, blsincl, secpkmsgs, secpkincl, nil } -type BlockSync struct { - bserv bserv.BlockService - host host.Host - - syncPeersLk sync.Mutex - syncPeers map[peer.ID]struct{} -} - -func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host) *BlockSync { - return &BlockSync{ - bserv: bserv, - host: h, - syncPeers: make(map[peer.ID]struct{}), - } -} - -func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error { - switch res.Status { - case 101: // Partial Response - panic("not handled") - case 201: // req.Start not found - return fmt.Errorf("not found") - case 202: // Go Away - panic("not handled") - case 203: // Internal Error - return fmt.Errorf("block sync peer errored: %s", res.Message) - case 204: - return fmt.Errorf("block sync request invalid: %s", res.Message) - default: - return fmt.Errorf("unrecognized response code: %d", res.Status) - } -} - -func (bs *BlockSync) GetBlocks(ctx context.Context, tipset []cid.Cid, count int) ([]*types.TipSet, error) { - ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks") - defer span.End() - if span.IsRecordingEvents() { - span.AddAttributes( - trace.StringAttribute("tipset", fmt.Sprint(tipset)), - trace.Int64Attribute("count", int64(count)), - ) - } - - peers := bs.getPeers() - perm := rand.Perm(len(peers)) - // TODO: round robin through these peers on error - - req := &BlockSyncRequest{ - Start: tipset, - RequestLength: uint64(count), - Options: BSOptBlocks, - } - - var oerr error - for _, p := range perm { - res, err := bs.sendRequestToPeer(ctx, peers[p], req) - if err != nil { - oerr = err - log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err) - continue - } - - if res.Status == 0 { - return bs.processBlocksResponse(req, res) - } - oerr = bs.processStatus(req, res) - if oerr != nil { - log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), oerr) - } - } - return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", oerr) -} - -func (bs *BlockSync) GetFullTipSet(ctx context.Context, p peer.ID, h []cid.Cid) (*store.FullTipSet, error) { - // TODO: round robin through these peers on error - - req := &BlockSyncRequest{ - Start: h, - RequestLength: 1, - Options: BSOptBlocks | BSOptMessages, - } - - res, err := bs.sendRequestToPeer(ctx, p, req) - if err != nil { - return nil, err - } - - switch res.Status { - case 0: // Success - if len(res.Chain) == 0 { - return nil, fmt.Errorf("got zero length chain response") - } - bts := res.Chain[0] - - return bstsToFullTipSet(bts) - case 101: // Partial Response - panic("not handled") - case 201: // req.Start not found - return nil, fmt.Errorf("not found") - case 202: // Go Away - panic("not handled") - case 203: // Internal Error - return nil, fmt.Errorf("block sync peer errored: %q", res.Message) - case 204: // Invalid Request - return nil, fmt.Errorf("block sync request invalid: %q", res.Message) - default: - return nil, fmt.Errorf("unrecognized response code") - } -} - -func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, count uint64) ([]*BSTipSet, error) { - ctx, span := trace.StartSpan(ctx, "GetChainMessages") - defer span.End() - - peers := bs.getPeers() - perm := rand.Perm(len(peers)) - // TODO: round robin through these peers on error - - req := &BlockSyncRequest{ - Start: h.Cids(), - RequestLength: count, - Options: BSOptMessages | BSOptBlocks, - } - - var err error - for _, p := range perm { - res, err := bs.sendRequestToPeer(ctx, peers[p], req) - if err != nil { - log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err) - continue - } - - if res.Status == 0 { - return res.Chain, nil - } - err = bs.processStatus(req, res) - if err != nil { - log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), err) - } - } - - // TODO: What if we have no peers (and err is nil)? - return nil, xerrors.Errorf("GetChainMessages failed with all peers(%d): %w", len(peers), err) -} - func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) { fts := &store.FullTipSet{} for i, b := range bts.Blocks { @@ -387,167 +235,3 @@ func bstsToFullTipSet(bts *BSTipSet) (*store.FullTipSet, error) { return fts, nil } - -func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) { - ctx, span := trace.StartSpan(ctx, "sendRequestToPeer") - defer span.End() - - if span.IsRecordingEvents() { - span.AddAttributes( - trace.StringAttribute("peer", p.Pretty()), - ) - } - - s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID) - if err != nil { - bs.RemovePeer(p) - return nil, err - } - - if err := cborutil.WriteCborRPC(s, req); err != nil { - return nil, err - } - - var res BlockSyncResponse - if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil { - return nil, err - } - - return &res, nil -} - -func (bs *BlockSync) processBlocksResponse(req *BlockSyncRequest, res *BlockSyncResponse) ([]*types.TipSet, error) { - cur, err := types.NewTipSet(res.Chain[0].Blocks) - if err != nil { - return nil, err - } - - out := []*types.TipSet{cur} - for bi := 1; bi < len(res.Chain); bi++ { - next := res.Chain[bi].Blocks - nts, err := types.NewTipSet(next) - if err != nil { - return nil, err - } - - if !types.CidArrsEqual(cur.Parents(), nts.Cids()) { - return nil, fmt.Errorf("parents of tipset[%d] were not tipset[%d]", bi-1, bi) - } - - out = append(out, nts) - cur = nts - } - return out, nil -} - -func (bs *BlockSync) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHeader, error) { - sb, err := bs.bserv.GetBlock(ctx, c) - if err != nil { - return nil, err - } - - return types.DecodeBlock(sb.RawData()) -} - -func (bs *BlockSync) AddPeer(p peer.ID) { - bs.syncPeersLk.Lock() - defer bs.syncPeersLk.Unlock() - bs.syncPeers[p] = struct{}{} -} - -func (bs *BlockSync) RemovePeer(p peer.ID) { - bs.syncPeersLk.Lock() - defer bs.syncPeersLk.Unlock() - delete(bs.syncPeers, p) -} - -func (bs *BlockSync) getPeers() []peer.ID { - bs.syncPeersLk.Lock() - defer bs.syncPeersLk.Unlock() - var out []peer.ID - for p := range bs.syncPeers { - out = append(out, p) - } - return out -} - -func (bs *BlockSync) logPeerQuality(p peer.ID) { - -} - -func (bs *BlockSync) FetchMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.Message, error) { - out := make([]*types.Message, len(cids)) - - err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error { - msg, err := types.DecodeMessage(b.RawData()) - if err != nil { - return err - } - - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - - out[i] = msg - return nil - }) - if err != nil { - return nil, err - } - return out, nil -} - -func (bs *BlockSync) FetchSignedMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.SignedMessage, error) { - out := make([]*types.SignedMessage, len(cids)) - - err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error { - smsg, err := types.DecodeSignedMessage(b.RawData()) - if err != nil { - return err - } - - if out[i] != nil { - return fmt.Errorf("received duplicate message") - } - - out[i] = smsg - return nil - }) - if err != nil { - return nil, err - } - return out, nil -} - -func (bs *BlockSync) fetchCids(ctx context.Context, cids []cid.Cid, cb func(int, blocks.Block) error) error { - resp := bs.bserv.GetBlocks(context.TODO(), cids) - - m := make(map[cid.Cid]int) - for i, c := range cids { - m[c] = i - } - - for i := 0; i < len(cids); i++ { - select { - case v, ok := <-resp: - if !ok { - if i == len(cids)-1 { - break - } - - return fmt.Errorf("failed to fetch all messages") - } - - ix, ok := m[v.Cid()] - if !ok { - return fmt.Errorf("received message we didnt ask for") - } - - if err := cb(ix, v); err != nil { - return err - } - } - } - - return nil -} diff --git a/chain/blocksync/blocksync_client.go b/chain/blocksync/blocksync_client.go new file mode 100644 index 000000000..e17263362 --- /dev/null +++ b/chain/blocksync/blocksync_client.go @@ -0,0 +1,400 @@ +package blocksync + +import ( + "bufio" + "context" + "fmt" + "math/rand" + "sort" + "sync" + "time" + + blocks "github.com/ipfs/go-block-format" + bserv "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + inet "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + host "github.com/libp2p/go-libp2p-host" + "go.opencensus.io/trace" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/cborutil" + "github.com/filecoin-project/lotus/node/modules/dtypes" +) + +type BlockSync struct { + bserv bserv.BlockService + host host.Host + + syncPeersLk sync.Mutex + syncPeers *bsPeerTracker +} + +func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host) *BlockSync { + return &BlockSync{ + bserv: bserv, + host: h, + syncPeers: newPeerTracker(), + } +} + +func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error { + switch res.Status { + case 101: // Partial Response + panic("not handled") + case 201: // req.Start not found + return fmt.Errorf("not found") + case 202: // Go Away + panic("not handled") + case 203: // Internal Error + return fmt.Errorf("block sync peer errored: %s", res.Message) + case 204: + return fmt.Errorf("block sync request invalid: %s", res.Message) + default: + return fmt.Errorf("unrecognized response code: %d", res.Status) + } +} + +func (bs *BlockSync) GetBlocks(ctx context.Context, tipset []cid.Cid, count int) ([]*types.TipSet, error) { + ctx, span := trace.StartSpan(ctx, "bsync.GetBlocks") + defer span.End() + if span.IsRecordingEvents() { + span.AddAttributes( + trace.StringAttribute("tipset", fmt.Sprint(tipset)), + trace.Int64Attribute("count", int64(count)), + ) + } + + peers := bs.getPeers() + perm := rand.Perm(len(peers)) + // TODO: round robin through these peers on error + + req := &BlockSyncRequest{ + Start: tipset, + RequestLength: uint64(count), + Options: BSOptBlocks, + } + + var oerr error + for _, p := range perm { + res, err := bs.sendRequestToPeer(ctx, peers[p], req) + if err != nil { + oerr = err + log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err) + continue + } + + if res.Status == 0 { + return bs.processBlocksResponse(req, res) + } + oerr = bs.processStatus(req, res) + if oerr != nil { + log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), oerr) + } + } + return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", oerr) +} + +func (bs *BlockSync) GetFullTipSet(ctx context.Context, p peer.ID, h []cid.Cid) (*store.FullTipSet, error) { + // TODO: round robin through these peers on error + + req := &BlockSyncRequest{ + Start: h, + RequestLength: 1, + Options: BSOptBlocks | BSOptMessages, + } + + res, err := bs.sendRequestToPeer(ctx, p, req) + if err != nil { + return nil, err + } + + switch res.Status { + case 0: // Success + if len(res.Chain) == 0 { + return nil, fmt.Errorf("got zero length chain response") + } + bts := res.Chain[0] + + return bstsToFullTipSet(bts) + case 101: // Partial Response + return nil, xerrors.Errorf("partial responses are not handled") + case 201: // req.Start not found + return nil, fmt.Errorf("not found") + case 202: // Go Away + return nil, xerrors.Errorf("received 'go away' response peer") + case 203: // Internal Error + return nil, fmt.Errorf("block sync peer errored: %q", res.Message) + case 204: // Invalid Request + return nil, fmt.Errorf("block sync request invalid: %q", res.Message) + default: + return nil, fmt.Errorf("unrecognized response code") + } +} + +func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, count uint64) ([]*BSTipSet, error) { + ctx, span := trace.StartSpan(ctx, "GetChainMessages") + defer span.End() + + peers := bs.getPeers() + perm := rand.Perm(len(peers)) + // TODO: round robin through these peers on error + + req := &BlockSyncRequest{ + Start: h.Cids(), + RequestLength: count, + Options: BSOptMessages | BSOptBlocks, + } + + var err error + for _, p := range perm { + res, err := bs.sendRequestToPeer(ctx, peers[p], req) + if err != nil { + log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err) + continue + } + + if res.Status == 0 { + return res.Chain, nil + } + err = bs.processStatus(req, res) + if err != nil { + log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), err) + } + } + + // TODO: What if we have no peers (and err is nil)? + return nil, xerrors.Errorf("GetChainMessages failed with all peers(%d): %w", len(peers), err) +} + +func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) { + ctx, span := trace.StartSpan(ctx, "sendRequestToPeer") + defer span.End() + + if span.IsRecordingEvents() { + span.AddAttributes( + trace.StringAttribute("peer", p.Pretty()), + ) + } + + s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID) + if err != nil { + bs.RemovePeer(p) + return nil, err + } + + if err := cborutil.WriteCborRPC(s, req); err != nil { + return nil, err + } + + var res BlockSyncResponse + if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil { + return nil, err + } + + return &res, nil +} + +func (bs *BlockSync) processBlocksResponse(req *BlockSyncRequest, res *BlockSyncResponse) ([]*types.TipSet, error) { + cur, err := types.NewTipSet(res.Chain[0].Blocks) + if err != nil { + return nil, err + } + + out := []*types.TipSet{cur} + for bi := 1; bi < len(res.Chain); bi++ { + next := res.Chain[bi].Blocks + nts, err := types.NewTipSet(next) + if err != nil { + return nil, err + } + + if !types.CidArrsEqual(cur.Parents(), nts.Cids()) { + return nil, fmt.Errorf("parents of tipset[%d] were not tipset[%d]", bi-1, bi) + } + + out = append(out, nts) + cur = nts + } + return out, nil +} + +func (bs *BlockSync) GetBlock(ctx context.Context, c cid.Cid) (*types.BlockHeader, error) { + sb, err := bs.bserv.GetBlock(ctx, c) + if err != nil { + return nil, err + } + + return types.DecodeBlock(sb.RawData()) +} + +func (bs *BlockSync) AddPeer(p peer.ID) { + bs.syncPeers.addPeer(p) +} + +func (bs *BlockSync) RemovePeer(p peer.ID) { + bs.syncPeers.removePeer(p) +} + +func (bs *BlockSync) getPeers() []peer.ID { + return bs.syncPeers.prefSortedPeers() +} + +func (bs *BlockSync) FetchMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.Message, error) { + out := make([]*types.Message, len(cids)) + + err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error { + msg, err := types.DecodeMessage(b.RawData()) + if err != nil { + return err + } + + if out[i] != nil { + return fmt.Errorf("received duplicate message") + } + + out[i] = msg + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func (bs *BlockSync) FetchSignedMessagesByCids(ctx context.Context, cids []cid.Cid) ([]*types.SignedMessage, error) { + out := make([]*types.SignedMessage, len(cids)) + + err := bs.fetchCids(ctx, cids, func(i int, b blocks.Block) error { + smsg, err := types.DecodeSignedMessage(b.RawData()) + if err != nil { + return err + } + + if out[i] != nil { + return fmt.Errorf("received duplicate message") + } + + out[i] = smsg + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func (bs *BlockSync) fetchCids(ctx context.Context, cids []cid.Cid, cb func(int, blocks.Block) error) error { + resp := bs.bserv.GetBlocks(context.TODO(), cids) + + m := make(map[cid.Cid]int) + for i, c := range cids { + m[c] = i + } + + for i := 0; i < len(cids); i++ { + select { + case v, ok := <-resp: + if !ok { + if i == len(cids)-1 { + break + } + + return fmt.Errorf("failed to fetch all messages") + } + + ix, ok := m[v.Cid()] + if !ok { + return fmt.Errorf("received message we didnt ask for") + } + + if err := cb(ix, v); err != nil { + return err + } + } + } + + return nil +} + +type peerStats struct { + successes int + failures int + firstSeen time.Time +} + +type bsPeerTracker struct { + peers map[peer.ID]*peerStats + lk sync.Mutex +} + +func newPeerTracker() *bsPeerTracker { + return &bsPeerTracker{ + peers: make(map[peer.ID]*peerStats), + } +} +func (bpt *bsPeerTracker) addPeer(p peer.ID) { + bpt.lk.Lock() + defer bpt.lk.Unlock() + if _, ok := bpt.peers[p]; ok { + return + } + bpt.peers[p] = &peerStats{ + firstSeen: time.Now(), + } + +} + +func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID { + // TODO: this could probably be cached, but as long as its not too many peers, fine for now + bpt.lk.Lock() + defer bpt.lk.Unlock() + out := make([]peer.ID, 0, len(bpt.peers)) + for p := range bpt.peers { + out = append(out, p) + } + + sort.Slice(out, func(i, j int) bool { + pi := bpt.peers[out[i]] + pj := bpt.peers[out[j]] + if pi.successes > pj.successes { + return true + } + if pi.failures < pj.successes { + return true + } + return pi.firstSeen.Before(pj.firstSeen) + }) + + return out +} + +func (bpt *bsPeerTracker) logSuccess(p peer.ID) { + bpt.lk.Lock() + defer bpt.lk.Unlock() + if pi, ok := bpt.peers[p]; !ok { + log.Warn("log success called on peer not in tracker") + return + } else { + pi.successes++ + } +} + +func (bpt *bsPeerTracker) logFailure(p peer.ID) { + bpt.lk.Lock() + defer bpt.lk.Unlock() + if pi, ok := bpt.peers[p]; !ok { + log.Warn("log failure called on peer not in tracker") + return + } else { + pi.failures++ + } +} + +func (bpt *bsPeerTracker) removePeer(p peer.ID) { + bpt.lk.Lock() + defer bpt.lk.Unlock() + delete(bpt.peers, p) +} diff --git a/go.mod b/go.mod index 8c16260b0..5197ee9df 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/libp2p/go-libp2p-connmgr v0.1.0 github.com/libp2p/go-libp2p-core v0.2.2 github.com/libp2p/go-libp2p-discovery v0.1.0 + github.com/libp2p/go-libp2p-host v0.1.0 github.com/libp2p/go-libp2p-kad-dht v0.1.1 github.com/libp2p/go-libp2p-mplex v0.2.1 github.com/libp2p/go-libp2p-peer v0.2.0 diff --git a/go.sum b/go.sum index 545d4d988..e6ae50214 100644 --- a/go.sum +++ b/go.sum @@ -305,6 +305,8 @@ github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoA github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= +github.com/libp2p/go-libp2p-host v0.1.0 h1:OZwENiFm6JOK3YR5PZJxkXlJE8a5u8g4YvAUrEV2MjM= +github.com/libp2p/go-libp2p-host v0.1.0/go.mod h1:5+fWuLbDn8OxoxPN3CV0vsLe1hAKScSMbT84qRfxum8= github.com/libp2p/go-libp2p-kad-dht v0.1.1 h1:IH6NQuoUv5w5e1O8Jc3KyVDtr0rNd0G9aaADpLI1xVo= github.com/libp2p/go-libp2p-kad-dht v0.1.1/go.mod h1:1kj2Rk5pX3/0RwqMm9AMNCT7DzcMHYhgDN5VTi+cY0M= github.com/libp2p/go-libp2p-kbucket v0.2.0 h1:FB2a0VkOTNGTP5gu/I444u4WabNM9V1zCkQcWb7zajI= From dbc706b846a2865ee1684c0fe44a85ad1b26f309 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sun, 10 Nov 2019 15:06:06 -0800 Subject: [PATCH 227/230] handle marking blocks as bad better --- chain/blocksync/blocksync_client.go | 16 ++++++-------- chain/sync.go | 33 ++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/chain/blocksync/blocksync_client.go b/chain/blocksync/blocksync_client.go index e17263362..6af46a592 100644 --- a/chain/blocksync/blocksync_client.go +++ b/chain/blocksync/blocksync_client.go @@ -67,22 +67,20 @@ func (bs *BlockSync) GetBlocks(ctx context.Context, tipset []cid.Cid, count int) ) } - peers := bs.getPeers() - perm := rand.Perm(len(peers)) - // TODO: round robin through these peers on error - req := &BlockSyncRequest{ Start: tipset, RequestLength: uint64(count), Options: BSOptBlocks, } + peers := bs.getPeers() + var oerr error - for _, p := range perm { - res, err := bs.sendRequestToPeer(ctx, peers[p], req) + for _, p := range peers { + res, err := bs.sendRequestToPeer(ctx, p, req) if err != nil { oerr = err - log.Warnf("BlockSync request failed for peer %s: %s", peers[p].String(), err) + log.Warnf("BlockSync request failed for peer %s: %s", p.String(), err) continue } @@ -91,7 +89,7 @@ func (bs *BlockSync) GetBlocks(ctx context.Context, tipset []cid.Cid, count int) } oerr = bs.processStatus(req, res) if oerr != nil { - log.Warnf("BlockSync peer %s response was an error: %s", peers[p].String(), oerr) + log.Warnf("BlockSync peer %s response was an error: %s", p.String(), oerr) } } return nil, xerrors.Errorf("GetBlocks failed with all peers: %w", oerr) @@ -182,7 +180,7 @@ func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *Bloc s, err := bs.host.NewStream(inet.WithNoDial(ctx, "should already have connection"), p, BlockSyncProtocolID) if err != nil { bs.RemovePeer(p) - return nil, err + return nil, xerrors.Errorf("failed to open stream to peer: %w", err) } if err := cborutil.WriteCborRPC(s, req); err != nil { diff --git a/chain/sync.go b/chain/sync.go index e7384e02f..d0e24c7a0 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -119,14 +119,15 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) { syncer.Bsync.AddPeer(from) bestPweight := syncer.store.GetHeaviestTipSet().Blocks()[0].ParentWeight - if fts.TipSet().Blocks()[0].ParentWeight.LessThan(bestPweight) { + targetWeight := fts.TipSet().Blocks()[0].ParentWeight + if targetWeight.LessThan(bestPweight) { log.Warn("incoming tipset does not appear to be better than our best chain, ignoring for now") return } go func() { if err := syncer.Sync(ctx, fts.TipSet()); err != nil { - log.Errorf("sync error: %+v", err) + log.Errorf("sync error (curW=%s, targetW=%s): %+v", bestPweight, targetWeight, err) } }() } @@ -358,6 +359,12 @@ func (syncer *Syncer) tryLoadFullTipSet(cids []cid.Cid) (*store.FullTipSet, erro func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { ctx, span := trace.StartSpan(ctx, "chain.Sync") defer span.End() + if span.IsRecordingEvents() { + span.AddAttributes( + trace.StringAttribute("tipset", fmt.Sprint(maybeHead.Cids())), + trace.Int64Attribute("height", int64(maybeHead.Height())), + ) + } syncer.syncLock.Lock() defer syncer.syncLock.Unlock() @@ -367,10 +374,12 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error { } if err := syncer.collectChain(ctx, maybeHead); err != nil { + span.AddAttributes(trace.StringAttribute("col_error", err.Error())) return xerrors.Errorf("collectChain failed: %w", err) } if err := syncer.store.PutTipSet(ctx, maybeHead); err != nil { + span.AddAttributes(trace.StringAttribute("put_error", err.Error())) return xerrors.Errorf("failed to put synced tipset to chainstore: %w", err) } @@ -690,6 +699,15 @@ func (syncer *Syncer) collectHeaders(ctx context.Context, from *types.TipSet, to trace.Int64Attribute("toHeight", int64(to.Height())), ) + for _, pcid := range from.Parents() { + if syncer.bad.Has(pcid) { + for _, b := range from.Cids() { + syncer.bad.Add(b) + } + return nil, xerrors.Errorf("chain linked to block marked previously as bad (%s, %s)", from.Cids(), pcid) + } + } + blockSet := []*types.TipSet{from} at := from.Parents() @@ -766,6 +784,13 @@ loop: log.Warnf("(fork detected) synced header chain (%s - %d) does not link to our best block (%s - %d)", from.Cids(), from.Height(), to.Cids(), to.Height()) fork, err := syncer.syncFork(ctx, last, to) if err != nil { + if xerrors.Is(err, ErrForkTooLong) { + // TODO: we're marking this block bad in the same way that we mark invalid blocks bad. Maybe distinguish? + log.Warn("adding forked chain to our bad tipset cache") + for _, b := range from.Blocks() { + syncer.bad.Add(b.Cid()) + } + } return nil, xerrors.Errorf("failed to sync fork: %w", err) } @@ -775,6 +800,8 @@ loop: return blockSet, nil } +var ErrForkTooLong = fmt.Errorf("fork longer than threshold") + func (syncer *Syncer) syncFork(ctx context.Context, from *types.TipSet, to *types.TipSet) ([]*types.TipSet, error) { tips, err := syncer.Bsync.GetBlocks(ctx, from.Parents(), build.ForkLengthThreshold) if err != nil { @@ -801,7 +828,7 @@ func (syncer *Syncer) syncFork(ctx context.Context, from *types.TipSet, to *type } } } - return nil, xerrors.Errorf("fork was longer than our threshold") + return nil, ErrForkTooLong } func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*types.TipSet) error { From b8e18ed7730e9ab9a3bdc679aee41d03b07c42c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 9 Nov 2019 13:11:46 +0100 Subject: [PATCH 228/230] storageminer: Don't print PP in info cmd if not proving --- cmd/lotus-storage-miner/info.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index aed867c1e..5b9b2b6a1 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -64,13 +64,18 @@ var infoCmd = &cli.Command{ if err != nil { return err } - head, err := api.ChainHead(ctx) - if err != nil { - return err + if ppe != 0 { + head, err := api.ChainHead(ctx) + if err != nil { + return err + } + pdiff := int64(ppe - head.Height()) + pdifft := pdiff * build.BlockDelay + fmt.Printf("Proving Period: %d, in %d Blocks (~%dm %ds)\n", ppe, pdiff, pdifft/60, pdifft%60) + } else { + fmt.Printf("Proving Period: Not Proving\n") } - pdiff := int64(ppe-head.Height()) - pdifft := pdiff * build.BlockDelay - fmt.Printf("Proving Period: %d, in %d Blocks (~%dm %ds)\n", ppe, pdiff, pdifft / 60, pdifft % 60) + sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { From e48f016b29532849f33fde511bd14895cdd0dc22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Nov 2019 12:11:45 +0100 Subject: [PATCH 229/230] Update sectorbuilder --- extern/go-sectorbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/go-sectorbuilder b/extern/go-sectorbuilder index 7c892757e..d94675a70 160000 --- a/extern/go-sectorbuilder +++ b/extern/go-sectorbuilder @@ -1 +1 @@ -Subproject commit 7c892757e0465fbdb38c342c4ea890775f63a59a +Subproject commit d94675a704d5f4c4bed30ec16538352f4d8cdc4b From b8f46bff2645635fd24e9186e18428fe1ce7f2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 11 Nov 2019 18:32:46 +0100 Subject: [PATCH 230/230] storageminer: Drop sortedSectorInfo log --- cmd/lotus-storage-miner/info.go | 1 - storage/post.go | 3 --- 2 files changed, 4 deletions(-) diff --git a/cmd/lotus-storage-miner/info.go b/cmd/lotus-storage-miner/info.go index 5b9b2b6a1..dc449c66e 100644 --- a/cmd/lotus-storage-miner/info.go +++ b/cmd/lotus-storage-miner/info.go @@ -76,7 +76,6 @@ var infoCmd = &cli.Command{ fmt.Printf("Proving Period: Not Proving\n") } - sinfo, err := sectorsInfo(ctx, nodeApi) if err != nil { return err diff --git a/storage/post.go b/storage/post.go index 0bcd66bb5..0b4393642 100644 --- a/storage/post.go +++ b/storage/post.go @@ -193,9 +193,6 @@ func (p *post) runPost(ctx context.Context) error { var seed [32]byte copy(seed[:], p.r) - vals := p.sortedSectorInfo() - log.Infof("SSI: %+v", vals.Values()) - proof, err := p.m.sb.GeneratePoSt(p.sortedSectorInfo(), seed, faults) if err != nil { return xerrors.Errorf("running post failed: %w", err)
{k}{v.NodeName}{v.Height}{mnrs}
{k}{v.NodeName}{v.Weight}({bestw - v.Weight}){v.Height}({besth - v.Height}){mnrs}