cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #1057 from filecoin-project/feat/markets-extraction

Browse Source 
Markets Extraction
This commit is contained in:
Łukasz Magiera 2020-01-10 20:49:09 +01:00 committed by GitHub
commit b6fda30719
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 404 additions and 5526 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

784
chain/deals/cbor_gen.go
View File

@ -1,784 +0,0 @@
package deals
import (
"fmt"
"io"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
var _ = xerrors.Errorf
func (t *AskRequest) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Miner (address.Address) (struct)
if err := t.Miner.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *AskRequest) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Miner (address.Address) (struct)
{
if err := t.Miner.UnmarshalCBOR(br); err != nil {
return err
}
}
return nil
}
func (t *AskResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Ask (types.SignedStorageAsk) (struct)
if err := t.Ask.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *AskResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Ask (types.SignedStorageAsk) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Ask = new(types.SignedStorageAsk)
if err := t.Ask.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *Proposal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.DealProposal (actors.StorageDealProposal) (struct)
if err := t.DealProposal.MarshalCBOR(w); err != nil {
return err
}
// t.Piece (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Piece); err != nil {
return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
}
return nil
}
func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.DealProposal (actors.StorageDealProposal) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.DealProposal = new(actors.StorageDealProposal)
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
return err
}
}
}
// t.Piece (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
}
t.Piece = c
}
return nil
}
func (t *Response) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{132}); err != nil {
return err
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Message (string) (string)
if len(t.Message) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Message was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Message)); err != nil {
return err
}
// t.Proposal (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Proposal); err != nil {
return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
}
// t.PublishMessage (cid.Cid) (struct)
if t.PublishMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.PublishMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err)
}
}
return nil
}
func (t *Response) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Message (string) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Message = string(sval)
}
// t.Proposal (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
}
t.Proposal = c
}
// t.PublishMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err)
}
t.PublishMessage = &c
}
}
return nil
}
func (t *SignedResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Response (deals.Response) (struct)
if err := t.Response.MarshalCBOR(w); err != nil {
return err
}
// t.Signature (types.Signature) (struct)
if err := t.Signature.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Response (deals.Response) (struct)
{
if err := t.Response.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Signature (types.Signature) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Signature = new(types.Signature)
if err := t.Signature.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{136}); err != nil {
return err
}
// t.Data (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Data); err != nil {
return xerrors.Errorf("failed to write cid field t.Data: %w", err)
}
// t.PricePerEpoch (types.BigInt) (struct)
if err := t.PricePerEpoch.MarshalCBOR(w); err != nil {
return err
}
// t.ProposalExpiration (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil {
return err
}
// t.Duration (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil {
return err
}
// t.ProviderAddress (address.Address) (struct)
if err := t.ProviderAddress.MarshalCBOR(w); err != nil {
return err
}
// t.Client (address.Address) (struct)
if err := t.Client.MarshalCBOR(w); err != nil {
return err
}
// t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.MinerID (peer.ID) (string)
if len(t.MinerID) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.MinerID was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.MinerID)); err != nil {
return err
}
return nil
}
func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 8 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Data (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Data: %w", err)
}
t.Data = c
}
// t.PricePerEpoch (types.BigInt) (struct)
{
if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.ProposalExpiration (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ProposalExpiration = uint64(extra)
// t.Duration (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Duration = uint64(extra)
// t.ProviderAddress (address.Address) (struct)
{
if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Client (address.Address) (struct)
{
if err := t.Client.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.MinerID (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.MinerID = peer.ID(sval)
}
return nil
}
func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.ClientDeal (storagemarket.ClientDeal) (struct)
if err := t.ClientDeal.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.ClientDeal (storagemarket.ClientDeal) (struct)
{
if err := t.ClientDeal.UnmarshalCBOR(br); err != nil {
return err
}
}
return nil
}
func (t *MinerDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{135}); err != nil {
return err
}
// t.Client (peer.ID) (string)
if len(t.Client) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Client was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Client)); err != nil {
return err
}
// t.Proposal (actors.StorageDealProposal) (struct)
if err := t.Proposal.MarshalCBOR(w); err != nil {
return err
}
// t.ProposalCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Ref (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Ref); err != nil {
return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
return nil
}
func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 7 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Client (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Client = peer.ID(sval)
}
// t.Proposal (actors.StorageDealProposal) (struct)
{
if err := t.Proposal.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.ProposalCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
}
t.ProposalCid = c
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Ref (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
}
t.Ref = c
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
return nil
}
func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Proposal (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Proposal); err != nil {
return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
return nil
}
func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Proposal (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
}
t.Proposal = c
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
return nil
}

293
chain/deals/client.go
View File

@ -1,293 +0,0 @@
package deals
import (
"context"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/host"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storagemarket"
)
var log = logging.Logger("deals")
type ClientDeal struct {
storagemarket.ClientDeal
s inet.Stream
}
type Client struct {
h host.Host
// dataTransfer
// TODO: once the data transfer module is complete, the
// client will listen to events on the data transfer module
// Because we are using only a fake DAGService
// implementation, there's no validation or events on the client side
dataTransfer dtypes.ClientDataTransfer
dag dtypes.ClientDAG
discovery *discovery.Local
node storagemarket.StorageClientNode
deals *statestore.StateStore
conns map[cid.Cid]inet.Stream
incoming chan *ClientDeal
updated chan clientDealUpdate
stop chan struct{}
stopped chan struct{}
}
type clientDealUpdate struct {
newState api.DealState
id cid.Cid
err error
mut func(*ClientDeal)
}
func NewClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) *Client {
c := &Client{
h: h,
dataTransfer: dataTransfer,
dag: dag,
discovery: discovery,
node: scn,
deals: deals,
conns: map[cid.Cid]inet.Stream{},
incoming: make(chan *ClientDeal, 16),
updated: make(chan clientDealUpdate, 16),
stop: make(chan struct{}),
stopped: make(chan struct{}),
}
return c
}
func (c *Client) Run(ctx context.Context) {
go func() {
defer close(c.stopped)
for {
select {
case deal := <-c.incoming:
c.onIncoming(deal)
case update := <-c.updated:
c.onUpdated(ctx, update)
case <-c.stop:
return
}
}
}()
}
func (c *Client) onIncoming(deal *ClientDeal) {
log.Info("incoming deal")
if _, ok := c.conns[deal.ProposalCid]; ok {
log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid)
return
}
c.conns[deal.ProposalCid] = deal.s
if err := c.deals.Begin(deal.ProposalCid, deal); err != nil {
// We may have re-sent the proposal
log.Errorf("deal tracking failed: %s", err)
c.failDeal(deal.ProposalCid, err)
return
}
go func() {
c.updated <- clientDealUpdate{
newState: api.DealUnknown,
id: deal.ProposalCid,
err: nil,
}
}()
}
func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState])
var deal ClientDeal
err := c.deals.Mutate(update.id, func(d *ClientDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
}
deal = *d
return nil
})
if update.err != nil {
log.Errorf("deal %s failed: %s", update.id, update.err)
c.failDeal(update.id, update.err)
return
}
if err != nil {
c.failDeal(update.id, err)
return
}
switch update.newState {
case api.DealUnknown: // new
c.handle(ctx, deal, c.new, api.DealAccepted)
case api.DealAccepted:
c.handle(ctx, deal, c.accepted, api.DealStaged)
case api.DealStaged:
c.handle(ctx, deal, c.staged, api.DealSealing)
case api.DealSealing:
c.handle(ctx, deal, c.sealing, api.DealNoUpdate)
// TODO: DealComplete -> watch for faults, expiration, etc.
}
}
type ClientDealProposal struct {
Data cid.Cid
PricePerEpoch types.BigInt
ProposalExpiration uint64
Duration uint64
ProviderAddress address.Address
Client address.Address
MinerWorker address.Address
MinerID peer.ID
}
func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) {
amount := types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))
if err := c.node.EnsureFunds(ctx, p.Client, storagemarket.TokenAmount(amount)); err != nil {
return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err)
}
commP, pieceSize, err := c.commP(ctx, p.Data)
if err != nil {
return cid.Undef, xerrors.Errorf("computing commP failed: %w", err)
}
dealProposal := &actors.StorageDealProposal{
PieceRef: commP,
PieceSize: uint64(pieceSize),
Client: p.Client,
Provider: p.ProviderAddress,
ProposalExpiration: p.ProposalExpiration,
Duration: p.Duration,
StoragePricePerEpoch: p.PricePerEpoch,
StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc
}
if err := c.node.SignProposal(ctx, p.Client, dealProposal); err != nil {
return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err)
}
proposalNd, err := cborutil.AsIpld(dealProposal)
if err != nil {
return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err)
}
s, err := c.h.NewStream(ctx, p.MinerID, storagemarket.DealProtocolID)
if err != nil {
return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
}
proposal := &Proposal{
DealProposal: dealProposal,
Piece: p.Data,
}
if err := cborutil.WriteCborRPC(s, proposal); err != nil {
s.Reset()
return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err)
}
deal := &ClientDeal{
ClientDeal: storagemarket.ClientDeal{
ProposalCid: proposalNd.Cid(),
Proposal: *dealProposal,
State: api.DealUnknown,
Miner: p.MinerID,
MinerWorker: p.MinerWorker,
PayloadCid: p.Data,
},
s: s,
}
c.incoming <- deal
return deal.ProposalCid, c.discovery.AddPeer(p.Data, retrievalmarket.RetrievalPeer{
Address: dealProposal.Provider,
ID: deal.Miner,
})
}
func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) {
s, err := c.h.NewStream(ctx, p, storagemarket.AskProtocolID)
if err != nil {
return nil, xerrors.Errorf("failed to open stream to miner: %w", err)
}
req := &AskRequest{
Miner: a,
}
if err := cborutil.WriteCborRPC(s, req); err != nil {
return nil, xerrors.Errorf("failed to send ask request: %w", err)
}
var out AskResponse
if err := cborutil.ReadCborRPC(s, &out); err != nil {
return nil, xerrors.Errorf("failed to read ask response: %w", err)
}
if out.Ask == nil {
return nil, xerrors.Errorf("got no ask back")
}
if out.Ask.Ask.Miner != a {
return nil, xerrors.Errorf("got back ask for wrong miner")
}
if err := c.checkAskSignature(out.Ask); err != nil {
return nil, xerrors.Errorf("ask was not properly signed")
}
return out.Ask, nil
}
func (c *Client) List() ([]ClientDeal, error) {
var out []ClientDeal
if err := c.deals.List(&out); err != nil {
return nil, err
}
return out, nil
}
func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
var out ClientDeal
if err := c.deals.Get(d, &out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) Stop() {
close(c.stop)
<-c.stopped
}

97
chain/deals/client_states.go
View File

@ -1,97 +0,0 @@
package deals
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
)
type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error)
func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) {
go func() {
mut, err := cb(ctx, deal)
if err != nil {
next = api.DealError
}
if err == nil && next == api.DealNoUpdate {
return
}
select {
case c.updated <- clientDealUpdate{
newState: next,
id: deal.ProposalCid,
err: err,
mut: mut,
}:
case <-c.stop:
}
}()
}
func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
resp, err := c.readStorageDealResp(deal)
if err != nil {
return nil, err
}
// TODO: verify StorageDealSubmission
if err := c.disconnect(deal); err != nil {
return nil, err
}
/* data transfer happens */
if resp.State != api.DealAccepted {
return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
}
return func(info *ClientDeal) {
info.PublishMessage = resp.PublishMessage
}, nil
}
func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
log.Infow("DEAL ACCEPTED!")
dealId, err := c.node.ValidatePublishedDeal(ctx, deal.ClientDeal)
if err != nil {
return nil, err
}
return func(info *ClientDeal) {
info.DealID = dealId
}, nil
}
func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
// TODO: Maybe wait for pre-commit
return nil, nil
}
func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
cb := func(err error) {
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
err: err,
}:
case <-c.stop:
}
}
err := c.node.OnDealSectorCommitted(ctx, deal.Proposal.Provider, deal.DealID, cb)
return nil, err
}
func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error {
return c.node.ValidateAskSignature(ask)
}

118
chain/deals/client_storagemarket.go
View File

@ -1,118 +0,0 @@
package deals
// this file implements storagemarket.StorageClient
import (
"context"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storagemarket"
)
func (c *Client) ListProviders(ctx context.Context) (<-chan storagemarket.StorageProviderInfo, error) {
providers, err := c.node.ListStorageProviders(ctx)
if err != nil {
return nil, err
}
out := make(chan storagemarket.StorageProviderInfo)
go func() {
for _, p := range providers {
select {
case out <- *p:
case <-ctx.Done():
return
}
}
}()
return out, nil
}
func (c *Client) ListDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) {
return c.node.ListClientDeals(ctx, addr)
}
func (c *Client) ListInProgressDeals(ctx context.Context) ([]storagemarket.ClientDeal, error) {
deals, err := c.List()
if err != nil {
return nil, err
}
out := make([]storagemarket.ClientDeal, len(deals))
for k, v := range deals {
out[k] = storagemarket.ClientDeal{
ProposalCid: v.ProposalCid,
Proposal: v.Proposal,
State: v.State,
Miner: v.Miner,
MinerWorker: v.MinerWorker,
DealID: v.DealID,
PublishMessage: v.PublishMessage,
}
}
return out, nil
}
func (c *Client) GetInProgressDeal(ctx context.Context, cid cid.Cid) (storagemarket.ClientDeal, error) {
deals, err := c.ListInProgressDeals(ctx)
if err != nil {
return storagemarket.ClientDeal{}, err
}
for _, deal := range deals {
if deal.ProposalCid == cid {
return deal, nil
}
}
return storagemarket.ClientDeal{}, xerrors.Errorf("couldn't find client deal")
}
func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderInfo) (*storagemarket.StorageAsk, error) {
return c.QueryAsk(ctx, info.PeerID, info.Address)
}
func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price storagemarket.TokenAmount, collateral storagemarket.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) {
proposal := ClientDealProposal{
Data: payloadCid,
PricePerEpoch: types.BigInt(price),
ProposalExpiration: uint64(proposalExpiration),
Duration: uint64(duration),
Client: addr,
ProviderAddress: info.Address,
MinerWorker: info.Worker,
MinerID: info.PeerID,
}
proposalCid, err := c.Start(ctx, proposal)
result := &storagemarket.ProposeStorageDealResult{
ProposalCid: proposalCid,
}
return result, err
}
func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
balance, err := c.node.GetBalance(ctx, addr)
return balance, err
}
func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error {
return c.node.AddFunds(ctx, addr, amount)
}
var _ storagemarket.StorageClient = &Client{}

170
chain/deals/client_utils.go
View File

@ -1,170 +0,0 @@
package deals
import (
"context"
"runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func (c *Client) failDeal(id cid.Cid, cerr error) {
if cerr == nil {
_, f, l, _ := runtime.Caller(1)
cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
}
s, ok := c.conns[id]
if ok {
_ = s.Reset()
delete(c.conns, id)
}
// TODO: store in some sort of audit log
log.Errorf("deal %s failed: %+v", id, cerr)
}
func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) {
root, err := c.dag.Get(ctx, data)
if err != nil {
log.Errorf("failed to get file root for deal: %s", err)
return nil, 0, err
}
n, err := unixfile.NewUnixfsFile(ctx, c.dag, root)
if err != nil {
log.Errorf("cannot open unixfs file: %s", err)
return nil, 0, err
}
uf, ok := n.(files.File)
if !ok {
// TODO: we probably got directory, how should we handle this in unixfs mode?
return nil, 0, xerrors.New("unsupported unixfs type")
}
s, err := uf.Size()
if err != nil {
return nil, 0, err
}
pr, psize := padreader.New(uf, uint64(s))
commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize)
if err != nil {
return nil, 0, xerrors.Errorf("generating CommP: %w", err)
}
return commp[:], psize, nil
}
func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
s, ok := c.conns[deal.ProposalCid]
if !ok {
// TODO: Try to re-establish the connection using query protocol
return nil, xerrors.Errorf("no connection to miner")
}
var resp SignedResponse
if err := cborutil.ReadCborRPC(s, &resp); err != nil {
log.Errorw("failed to read Response message", "error", err)
return nil, err
}
if err := resp.Verify(deal.MinerWorker); err != nil {
return nil, xerrors.Errorf("verifying response signature failed", err)
}
if resp.Response.Proposal != deal.ProposalCid {
return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid)
}
return &resp.Response, nil
}
func (c *Client) disconnect(deal ClientDeal) error {
s, ok := c.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(c.conns, deal.ProposalCid)
return err
}
var _ datatransfer.RequestValidator = &ClientRequestValidator{}
// ClientRequestValidator validates data transfer requests for the client
// in a storage market
type ClientRequestValidator struct {
deals *statestore.StateStore
}
// NewClientRequestValidator returns a new client request validator for the
// given datastore
func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator {
crv := &ClientRequestValidator{
deals: deals,
}
return crv
}
// ValidatePush validates a push request received from the peer that will send data
// Will always error because clients should not accept push requests from a provider
// in a storage deal (i.e. send data to client).
func (c *ClientRequestValidator) ValidatePush(
sender peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
return ErrNoPushAccepted
}
// ValidatePull validates a pull request received from the peer that will receive data
// Will succeed only if:
// - voucher has correct type
// - voucher references an active deal
// - referenced deal matches the receiver (miner)
// - referenced deal matches the given base CID
// - referenced deal is in an acceptable state
func (c *ClientRequestValidator) ValidatePull(
receiver peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
if !ok {
return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType)
}
var deal ClientDeal
err := c.deals.Get(dealVoucher.Proposal, &deal)
if err != nil {
return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
}
if deal.Miner != receiver {
return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer)
}
if !deal.PayloadCid.Equals(baseCid) {
return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
}
for _, state := range DataTransferStates {
if deal.State == state {
return nil
}
}
return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
}

293
chain/deals/provider.go
View File

@ -1,293 +0,0 @@
package deals
import (
"context"
"errors"
"sync"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"github.com/libp2p/go-libp2p-core/host"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storagemarket"
)
var ProviderDsPrefix = "/deals/provider"
type MinerDeal struct {
Client peer.ID
Proposal actors.StorageDealProposal
ProposalCid cid.Cid
State api.DealState
Ref cid.Cid
DealID uint64
SectorID uint64 // Set when State >= DealStaged
s inet.Stream
}
type Provider struct {
pricePerByteBlock types.BigInt // how much we want for storing one byte for one block
minPieceSize uint64
ask *types.SignedStorageAsk
askLk sync.Mutex
spn storagemarket.StorageProviderNode
// TODO: This will go away once storage market module + CAR
// is implemented
dag dtypes.StagingDAG
// dataTransfer is the manager of data transfers used by this storage provider
dataTransfer dtypes.ProviderDataTransfer
deals *statestore.StateStore
ds dtypes.MetadataDS
conns map[cid.Cid]inet.Stream
actor address.Address
incoming chan MinerDeal
updated chan minerDealUpdate
stop chan struct{}
stopped chan struct{}
}
type minerDealUpdate struct {
newState api.DealState
id cid.Cid
err error
mut func(*MinerDeal)
}
var (
// ErrDataTransferFailed means a data transfer for a deal failed
ErrDataTransferFailed = errors.New("deal data transfer failed")
)
func NewProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) {
addr, err := ds.Get(datastore.NewKey("miner-address"))
if err != nil {
return nil, err
}
minerAddress, err := address.NewFromBytes(addr)
if err != nil {
return nil, err
}
h := &Provider{
dag: dag,
dataTransfer: dataTransfer,
spn: spn,
pricePerByteBlock: types.NewInt(3), // TODO: allow setting
minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up))
conns: map[cid.Cid]inet.Stream{},
incoming: make(chan MinerDeal),
updated: make(chan minerDealUpdate),
stop: make(chan struct{}),
stopped: make(chan struct{}),
actor: minerAddress,
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))),
ds: ds,
}
if err := h.tryLoadAsk(); err != nil {
return nil, err
}
if h.ask == nil {
// TODO: we should be fine with this state, and just say it means 'not actively accepting deals'
// for now... lets just set a price
if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil {
return nil, xerrors.Errorf("failed setting a default price: %w", err)
}
}
// register a data transfer event handler -- this will move deals from
// accepted to staged
h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent)
return h, nil
}
func (p *Provider) Run(ctx context.Context, host host.Host) {
// TODO: restore state
host.SetStreamHandler(storagemarket.DealProtocolID, p.HandleStream)
host.SetStreamHandler(storagemarket.AskProtocolID, p.HandleAskStream)
go func() {
defer log.Warn("quitting deal provider loop")
defer close(p.stopped)
for {
select {
case deal := <-p.incoming: // DealAccepted
p.onIncoming(deal)
case update := <-p.updated: // DealStaged
p.onUpdated(ctx, update)
case <-p.stop:
return
}
}
}()
}
func (p *Provider) onIncoming(deal MinerDeal) {
log.Info("incoming deal")
p.conns[deal.ProposalCid] = deal.s
if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil {
// This can happen when client re-sends proposal
p.failDeal(context.TODO(), deal.ProposalCid, err)
log.Errorf("deal tracking failed: %s", err)
return
}
go func() {
p.updated <- minerDealUpdate{
newState: api.DealAccepted,
id: deal.ProposalCid,
err: nil,
}
}()
}
func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState])
if update.err != nil {
log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err)
p.failDeal(ctx, update.id, update.err)
return
}
var deal MinerDeal
err := p.deals.Mutate(update.id, func(d *MinerDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
}
deal = *d
return nil
})
if err != nil {
p.failDeal(ctx, update.id, err)
return
}
switch update.newState {
case api.DealAccepted:
p.handle(ctx, deal, p.accept, api.DealNoUpdate)
case api.DealStaged:
p.handle(ctx, deal, p.staged, api.DealSealing)
case api.DealSealing:
p.handle(ctx, deal, p.sealing, api.DealComplete)
case api.DealComplete:
p.handle(ctx, deal, p.complete, api.DealNoUpdate)
}
}
// onDataTransferEvent is the function called when an event occurs in a data
// transfer -- it reads the voucher to verify this even occurred in a storage
// market deal, then, based on the data transfer event that occurred, it generates
// and update message for the deal -- either moving to staged for a completion
// event or moving to error if a data transfer error occurs
func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) {
voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher)
// if this event is for a transfer not related to storage, ignore
if !ok {
return
}
// data transfer events for opening and progress do not affect deal state
var next api.DealState
var err error
var mut func(*MinerDeal)
switch event.Code {
case datatransfer.Complete:
next = api.DealStaged
mut = func(deal *MinerDeal) {
deal.DealID = voucher.DealID
}
case datatransfer.Error:
next = api.DealFailed
err = ErrDataTransferFailed
default:
// the only events we care about are complete and error
return
}
select {
case p.updated <- minerDealUpdate{
newState: next,
id: voucher.Proposal,
err: err,
mut: mut,
}:
case <-p.stop:
}
}
func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
proposalNd, err := cborutil.AsIpld(proposal.DealProposal)
if err != nil {
return MinerDeal{}, err
}
return MinerDeal{
Client: s.Conn().RemotePeer(),
Proposal: *proposal.DealProposal,
ProposalCid: proposalNd.Cid(),
State: api.DealUnknown,
Ref: proposal.Piece,
s: s,
}, nil
}
func (p *Provider) HandleStream(s inet.Stream) {
log.Info("Handling storage deal proposal!")
proposal, err := p.readProposal(s)
if err != nil {
log.Error(err)
s.Close()
return
}
deal, err := p.newDeal(s, proposal)
if err != nil {
log.Errorf("%+v", err)
s.Close()
return
}
p.incoming <- deal
}
func (p *Provider) Stop() {
close(p.stop)
<-p.stopped
}

143
chain/deals/provider_asks.go
View File

@ -1,143 +0,0 @@
package deals
import (
"bytes"
"context"
"time"
"github.com/ipfs/go-datastore"
inet "github.com/libp2p/go-libp2p-core/network"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/types"
)
func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error {
p.askLk.Lock()
defer p.askLk.Unlock()
var seqno uint64
if p.ask != nil {
seqno = p.ask.Ask.SeqNo + 1
}
now := time.Now().Unix()
ask := &types.StorageAsk{
Price: price,
Timestamp: uint64(now),
Expiry: uint64(now + ttlsecs),
Miner: p.actor,
SeqNo: seqno,
MinPieceSize: p.minPieceSize,
}
ssa, err := p.signAsk(ask)
if err != nil {
return err
}
return p.saveAsk(ssa)
}
func (p *Provider) GetAsk(m address.Address) *types.SignedStorageAsk {
p.askLk.Lock()
defer p.askLk.Unlock()
if m != p.actor {
return nil
}
return p.ask
}
func (p *Provider) HandleAskStream(s inet.Stream) {
defer s.Close()
var ar AskRequest
if err := cborutil.ReadCborRPC(s, &ar); err != nil {
log.Errorf("failed to read AskRequest from incoming stream: %s", err)
return
}
resp := p.processAskRequest(&ar)
if err := cborutil.WriteCborRPC(s, resp); err != nil {
log.Errorf("failed to write ask response: %s", err)
return
}
}
func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse {
return &AskResponse{
Ask: p.GetAsk(ar.Miner),
}
}
var bestAskKey = datastore.NewKey("latest-ask")
func (p *Provider) tryLoadAsk() error {
p.askLk.Lock()
defer p.askLk.Unlock()
err := p.loadAsk()
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
log.Warn("no previous ask found, miner will not accept deals until a price is set")
return nil
}
return err
}
return nil
}
func (p *Provider) loadAsk() error {
askb, err := p.ds.Get(datastore.NewKey("latest-ask"))
if err != nil {
return xerrors.Errorf("failed to load most recent ask from disk: %w", err)
}
var ssa types.SignedStorageAsk
if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
return err
}
p.ask = &ssa
return nil
}
func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) {
b, err := cborutil.Dump(a)
if err != nil {
return nil, err
}
worker, err := p.spn.GetMinerWorker(context.TODO(), p.actor)
if err != nil {
return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err)
}
sig, err := p.spn.SignBytes(context.TODO(), worker, b)
if err != nil {
return nil, err
}
return &types.SignedStorageAsk{
Ask: a,
Signature: sig,
}, nil
}
func (p *Provider) saveAsk(a *types.SignedStorageAsk) error {
b, err := cborutil.Dump(a)
if err != nil {
return err
}
if err := p.ds.Put(bestAskKey, b); err != nil {
return err
}
p.ask = a
return nil
}

172
chain/deals/provider_states.go
View File

@ -1,172 +0,0 @@
package deals
import (
"context"
ipldfree "github.com/ipld/go-ipld-prime/impl/free"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storagemarket"
)
type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error)
func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) {
go func() {
mut, err := cb(ctx, deal)
if err == nil && next == api.DealNoUpdate {
return
}
select {
case p.updated <- minerDealUpdate{
newState: next,
id: deal.ProposalCid,
err: err,
mut: mut,
}:
case <-p.stop:
}
}()
}
// ACCEPTED
func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
head, err := p.spn.MostRecentStateId(ctx)
if err != nil {
return nil, err
}
if head.Height() >= deal.Proposal.ProposalExpiration {
return nil, xerrors.Errorf("deal proposal already expired")
}
// TODO: check StorageCollateral
minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30))
if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) {
return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice)
}
if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize {
return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize)
}
// check market funds
clientMarketBalance, err := p.spn.GetBalance(ctx, deal.Proposal.Client)
if err != nil {
return nil, xerrors.Errorf("getting client market balance failed: %w", err)
}
// This doesn't guarantee that the client won't withdraw / lock those funds
// but it's a decent first filter
if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) {
return nil, xerrors.New("clientMarketBalance.Available too small")
}
waddr, err := p.spn.GetMinerWorker(ctx, deal.Proposal.Provider)
if err != nil {
return nil, err
}
// TODO: check StorageCollateral (may be too large (or too small))
if err := p.spn.EnsureFunds(ctx, waddr, storagemarket.TokenAmount(deal.Proposal.StorageCollateral)); err != nil {
return nil, err
}
smDeal := storagemarket.MinerDeal{
Client: deal.Client,
Proposal: deal.Proposal,
ProposalCid: deal.ProposalCid,
State: deal.State,
Ref: deal.Ref,
SectorID: deal.SectorID,
}
dealId, mcid, err := p.spn.PublishDeals(ctx, smDeal)
if err != nil {
return nil, err
}
log.Infof("fetching data for a deal %d", dealId)
err = p.sendSignedResponse(ctx, &Response{
State: api.DealAccepted,
Proposal: deal.ProposalCid,
PublishMessage: &mcid,
})
if err != nil {
return nil, err
}
if err := p.disconnect(deal); err != nil {
log.Warnf("closing client connection: %+v", err)
}
ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder())
// this is the selector for "get the whole DAG"
// TODO: support storage deals with custom payload selectors
allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
// initiate a pull data transfer. This will complete asynchronously and the
// completion of the data transfer will trigger a change in deal state
// (see onDataTransferEvent)
_, err = p.dataTransfer.OpenPullDataChannel(ctx,
deal.Client,
&StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: uint64(dealId)},
deal.Ref,
allSelector,
)
if err != nil {
return nil, xerrors.Errorf("failed to open pull data channel: %w", err)
}
return nil, nil
}
// STAGED
func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
sectorID, err := p.spn.OnDealComplete(
ctx,
storagemarket.MinerDeal{
Client: deal.Client,
Proposal: deal.Proposal,
ProposalCid: deal.ProposalCid,
State: deal.State,
Ref: deal.Ref,
DealID: deal.DealID,
},
"",
)
if err != nil {
return nil, err
}
return func(deal *MinerDeal) {
deal.SectorID = sectorID
}, nil
}
// SEALING
func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
// TODO: consider waiting for seal to happen
return nil, nil
}
func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
// TODO: observe sector lifecycle, status, expiration..
return nil, nil
}

65
chain/deals/provider_storagemarket.go
View File

@ -1,65 +0,0 @@
package deals
// this file implements storagemarket.StorageClient
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storagemarket"
)
func (p *Provider) AddAsk(price storagemarket.TokenAmount, ttlsecs int64) error {
return p.SetPrice(types.BigInt(price), ttlsecs)
}
func (p *Provider) ListAsks(addr address.Address) []*types.SignedStorageAsk {
ask := p.GetAsk(addr)
if ask != nil {
return []*types.SignedStorageAsk{ask}
}
return nil
}
func (p *Provider) ListDeals(ctx context.Context) ([]actors.OnChainDeal, error) {
return p.spn.ListProviderDeals(ctx, p.actor)
}
func (p *Provider) AddStorageCollateral(ctx context.Context, amount storagemarket.TokenAmount) error {
return p.spn.AddFunds(ctx, p.actor, amount)
}
func (p *Provider) GetStorageCollateral(ctx context.Context) (storagemarket.Balance, error) {
balance, err := p.spn.GetBalance(ctx, p.actor)
return balance, err
}
func (p *Provider) ListIncompleteDeals() ([]storagemarket.MinerDeal, error) {
var out []storagemarket.MinerDeal
var deals []MinerDeal
if err := p.deals.List(&deals); err != nil {
return nil, err
}
for _, deal := range deals {
out = append(out, storagemarket.MinerDeal{
Client: deal.Client,
Proposal: deal.Proposal,
ProposalCid: deal.ProposalCid,
State: deal.State,
Ref: deal.Ref,
DealID: deal.DealID,
SectorID: deal.SectorID,
})
}
return out, nil
}
var _ storagemarket.StorageProvider = &Provider{}

180
chain/deals/provider_utils.go
View File

@ -1,180 +0,0 @@
package deals
import (
"context"
"runtime"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-statestore"
"github.com/ipfs/go-cid"
"github.com/ipld/go-ipld-prime"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func (p *Provider) failDeal(ctx context.Context, id cid.Cid, cerr error) {
if err := p.deals.End(id); err != nil {
log.Warnf("deals.End: %s", err)
}
if cerr == nil {
_, f, l, _ := runtime.Caller(1)
cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
}
log.Warnf("deal %s failed: %s", id, cerr)
err := p.sendSignedResponse(ctx, &Response{
State: api.DealFailed,
Message: cerr.Error(),
Proposal: id,
})
s, ok := p.conns[id]
if ok {
_ = s.Reset()
delete(p.conns, id)
}
if err != nil {
log.Warnf("notifying client about deal failure: %s", err)
}
}
func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) {
if err := cborutil.ReadCborRPC(s, &proposal); err != nil {
log.Errorw("failed to read proposal message", "error", err)
return proposal, err
}
if proposal.DealProposal.ProposerSignature == nil {
return proposal, xerrors.Errorf("incoming deal proposal has no signature")
}
if err := proposal.DealProposal.Verify(address.Undef); err != nil {
return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err)
}
if proposal.DealProposal.Provider != p.actor {
log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider)
return proposal, err
}
return
}
func (p *Provider) sendSignedResponse(ctx context.Context, resp *Response) error {
s, ok := p.conns[resp.Proposal]
if !ok {
return xerrors.New("couldn't send response: not connected")
}
msg, err := cborutil.Dump(resp)
if err != nil {
return xerrors.Errorf("serializing response: %w", err)
}
worker, err := p.spn.GetMinerWorker(ctx, p.actor)
if err != nil {
return err
}
sig, err := p.spn.SignBytes(ctx, worker, msg)
if err != nil {
return xerrors.Errorf("failed to sign response message: %w", err)
}
signedResponse := &SignedResponse{
Response: *resp,
Signature: sig,
}
err = cborutil.WriteCborRPC(s, signedResponse)
if err != nil {
// Assume client disconnected
s.Close()
delete(p.conns, resp.Proposal)
}
return err
}
func (p *Provider) disconnect(deal MinerDeal) error {
s, ok := p.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(p.conns, deal.ProposalCid)
return err
}
var _ datatransfer.RequestValidator = &ProviderRequestValidator{}
// ProviderRequestValidator validates data transfer requests for the provider
// in a storage market
type ProviderRequestValidator struct {
deals *statestore.StateStore
}
// NewProviderRequestValidator returns a new client request validator for the
// given datastore
func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator {
return &ProviderRequestValidator{
deals: deals,
}
}
// ValidatePush validates a push request received from the peer that will send data
// Will succeed only if:
// - voucher has correct type
// - voucher references an active deal
// - referenced deal matches the client
// - referenced deal matches the given base CID
// - referenced deal is in an acceptable state
func (m *ProviderRequestValidator) ValidatePush(
sender peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
if !ok {
return xerrors.Errorf("voucher type %s: %w", voucher.Type(), ErrWrongVoucherType)
}
var deal MinerDeal
err := m.deals.Get(dealVoucher.Proposal, &deal)
if err != nil {
return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
}
if deal.Client != sender {
return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer)
}
if !deal.Ref.Equals(baseCid) {
return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
}
for _, state := range DataTransferStates {
if deal.State == state {
return nil
}
}
return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
}
// ValidatePull validates a pull request received from the peer that will receive data.
// Will always error because providers should not accept pull requests from a client
// in a storage deal (i.e. send data to client).
func (m *ProviderRequestValidator) ValidatePull(
receiver peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
return ErrNoPullAccepted
}

274
chain/deals/request_validation_test.go
View File

@ -1,274 +0,0 @@
package deals_test
import (
"fmt"
"math/rand"
"testing"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
dss "github.com/ipfs/go-datastore/sync"
blocksutil "github.com/ipfs/go-ipfs-blocksutil"
"github.com/libp2p/go-libp2p-core/peer"
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storagemarket"
)
var blockGenerator = blocksutil.NewBlockGenerator()
type wrongDTType struct {
}
func (wrongDTType) ToBytes() ([]byte, error) {
return []byte{}, nil
}
func (wrongDTType) FromBytes([]byte) error {
return fmt.Errorf("not implemented")
}
func (wrongDTType) Type() string {
return "WrongDTTYPE"
}
func uniqueStorageDealProposal() (actors.StorageDealProposal, error) {
clientAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return actors.StorageDealProposal{}, err
}
providerAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return actors.StorageDealProposal{}, err
}
return actors.StorageDealProposal{
PieceRef: blockGenerator.Next().Cid().Bytes(),
Client: clientAddr,
Provider: providerAddr,
ProposerSignature: &types.Signature{
Data: []byte("foo bar cat dog"),
Type: types.KTBLS,
},
}, nil
}
func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) {
newProposal, err := uniqueStorageDealProposal()
if err != nil {
return deals.ClientDeal{}, err
}
proposalNd, err := cborutil.AsIpld(&newProposal)
if err != nil {
return deals.ClientDeal{}, err
}
minerAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return deals.ClientDeal{}, err
}
return deals.ClientDeal{
ClientDeal: storagemarket.ClientDeal{
Proposal: newProposal,
ProposalCid: proposalNd.Cid(),
PayloadCid: blockGenerator.Next().Cid(),
Miner: minerID,
MinerWorker: minerAddr,
State: state,
},
}, nil
}
func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) {
newProposal, err := uniqueStorageDealProposal()
if err != nil {
return deals.MinerDeal{}, err
}
proposalNd, err := cborutil.AsIpld(&newProposal)
if err != nil {
return deals.MinerDeal{}, err
}
ref := blockGenerator.Next().Cid()
return deals.MinerDeal{
Proposal: newProposal,
ProposalCid: proposalNd.Cid(),
Client: clientID,
State: state,
Ref: ref,
}, nil
}
func TestClientRequestValidation(t *testing.T) {
ds := dss.MutexWrap(datastore.NewMapDatastore())
state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
crv := deals.NewClientRequestValidator(state)
minerID := peer.ID("fakepeerid")
block := blockGenerator.Next()
t.Run("ValidatePush fails", func(t *testing.T) {
if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) {
t.Fatal("Push should fail for the client request validator for storage deals")
}
})
t.Run("ValidatePull fails deal not found", func(t *testing.T) {
proposal, err := uniqueStorageDealProposal()
if err != nil {
t.Fatal("error creating proposal")
}
proposalNd, err := cborutil.AsIpld(&proposal)
if err != nil {
t.Fatal("error serializing proposal")
}
pieceRef, err := cid.Cast(proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Pull should fail if there is no deal stored")
}
})
t.Run("ValidatePull fails wrong client", func(t *testing.T) {
otherMiner := peer.ID("otherminer")
clientDeal, err := newClientDeal(otherMiner, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
payloadCid := clientDeal.PayloadCid
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrWrongPeer) {
t.Fatal("Pull should fail if miner address is incorrect")
}
})
t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Pull should fail if piece ref is incorrect")
}
})
t.Run("ValidatePull fails wrong deal state", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealComplete)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
payloadCid := clientDeal.PayloadCid
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil), deals.ErrInacceptableDealState) {
t.Fatal("Pull should fail if deal is in a state that cannot be data transferred")
}
})
t.Run("ValidatePull succeeds", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
payloadCid := clientDeal.PayloadCid
if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, payloadCid, nil) != nil {
t.Fatal("Pull should should succeed when all parameters are correct")
}
})
}
func TestProviderRequestValidation(t *testing.T) {
ds := dss.MutexWrap(datastore.NewMapDatastore())
state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
mrv := deals.NewProviderRequestValidator(state)
clientID := peer.ID("fakepeerid")
block := blockGenerator.Next()
t.Run("ValidatePull fails", func(t *testing.T) {
if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) {
t.Fatal("Pull should fail for the provider request validator for storage deals")
}
})
t.Run("ValidatePush fails deal not found", func(t *testing.T) {
proposal, err := uniqueStorageDealProposal()
if err != nil {
t.Fatal("error creating proposal")
}
proposalNd, err := cborutil.AsIpld(&proposal)
if err != nil {
t.Fatal("error serializing proposal")
}
pieceRef, err := cid.Cast(proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Push should fail if there is no deal stored")
}
})
t.Run("ValidatePush fails wrong miner", func(t *testing.T) {
otherClient := peer.ID("otherclient")
minerDeal, err := newMinerDeal(otherClient, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
ref := minerDeal.Ref
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrWrongPeer) {
t.Fatal("Push should fail if miner address is incorrect")
}
})
t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Push should fail if piece ref is incorrect")
}
})
t.Run("ValidatePush fails wrong deal state", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealComplete)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
ref := minerDeal.Ref
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil), deals.ErrInacceptableDealState) {
t.Fatal("Push should fail if deal is in a state that cannot be data transferred")
}
})
t.Run("ValidatePush succeeds", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
ref := minerDeal.Ref
if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, ref, nil) != nil {
t.Fatal("Push should should succeed when all parameters are correct")
}
})
}

112
chain/deals/types.go
View File

@ -1,112 +0,0 @@
package deals
import (
"bytes"
"errors"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
)
var (
// ErrWrongVoucherType means the voucher was not the correct type can validate against
ErrWrongVoucherType = errors.New("cannot validate voucher type.")
// ErrNoPushAccepted just means clients do not accept pushes for storage deals
ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.")
// ErrNoPullAccepted just means providers do not accept pulls for storage deals
ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.")
// ErrNoDeal means no active deal was found for this vouchers proposal cid
ErrNoDeal = errors.New("no deal found for this proposal.")
// ErrWrongPeer means that the other peer for this data transfer request does not match
// the other peer for the deal
ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.")
// ErrWrongPiece means that the pieceref for this data transfer request does not match
// the one specified in the deal
ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.")
// ErrInacceptableDealState means the deal for this transfer is not in a deal state
// where transfer can be performed
ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.")
// DataTransferStates are the states in which it would make sense to actually start a data transfer
DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown}
)
type Proposal struct {
DealProposal *actors.StorageDealProposal
Piece cid.Cid // Used for retrieving from the client
}
type Response struct {
State api.DealState
// DealProposalRejected
Message string
Proposal cid.Cid
// DealAccepted
PublishMessage *cid.Cid
}
// TODO: Do we actually need this to be signed?
type SignedResponse struct {
Response Response
Signature *types.Signature
}
func (r *SignedResponse) Verify(addr address.Address) error {
b, err := cborutil.Dump(&r.Response)
if err != nil {
return err
}
return r.Signature.Verify(addr, b)
}
type AskRequest struct {
Miner address.Address
}
type AskResponse struct {
Ask *types.SignedStorageAsk
}
// StorageDataTransferVoucher is the voucher type for data transfers
// used by the storage market
type StorageDataTransferVoucher struct {
Proposal cid.Cid
DealID uint64
}
// ToBytes converts the StorageDataTransferVoucher to raw bytes
func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) {
var buf bytes.Buffer
err := dv.MarshalCBOR(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// FromBytes converts the StorageDataTransferVoucher to raw bytes
func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error {
r := bytes.NewReader(raw)
return dv.UnmarshalCBOR(r)
}
// Type is the unique string identifier for a StorageDataTransferVoucher
func (dv *StorageDataTransferVoucher) Type() string {
return "StorageDataTransferVoucher"
}

28
cmd/lotus-storage-miner/init.go
View File

@ -21,15 +21,17 @@ import (
"gopkg.in/urfave/cli.v2" "gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util" cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-sectorbuilder"
lapi "github.com/filecoin-project/lotus/api" lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli" lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
@ -281,15 +283,21 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String())
proposal, err := utils.ToSharedStorageDealProposal(&sector.Deal)
if err != nil {
return err
}
deal := &deals.MinerDeal{ deal := &deals.MinerDeal{
Proposal: sector.Deal, MinerDeal: storagemarket.MinerDeal{
ProposalCid: proposalCid, Proposal: *proposal,
State: lapi.DealComplete, ProposalCid: proposalCid,
Ref: proposalCid, // TODO: This is super wrong, but there State: lapi.DealComplete,
// are no params for CommP CIDs, we can't recover unixfs cid easily, Ref: proposalCid, // TODO: This is super wrong, but there
// and this isn't even used after the deal enters Complete state // are no params for CommP CIDs, we can't recover unixfs cid easily,
DealID: dealID, // and this isn't even used after the deal enters Complete state
SectorID: sector.SectorID, DealID: dealID,
SectorID: sector.SectorID,
},
} }
b, err = cborutil.Dump(deal) b, err = cborutil.Dump(deal)

27
gen/main.go
View File

@ -2,7 +2,6 @@ package main
import ( import (
"fmt" "fmt"
"github.com/filecoin-project/lotus/storagemarket"
"os" "os"
gen "github.com/whyrusleeping/cbor-gen" gen "github.com/whyrusleeping/cbor-gen"
@ -10,7 +9,6 @@ import (
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage"
@ -123,31 +121,6 @@ func main() {
os.Exit(1) os.Exit(1)
} }
err = gen.WriteTupleEncodersToFile("./storagemarket/cbor_gen.go", "storagemarket",
storagemarket.ClientDeal{},
storagemarket.MinerDeal{},
)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals",
deals.AskRequest{},
deals.AskResponse{},
deals.Proposal{},
deals.Response{},
deals.SignedResponse{},
deals.ClientDealProposal{},
deals.ClientDeal{},
deals.MinerDeal{},
deals.StorageDataTransferVoucher{},
)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage", err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage",
storage.SealTicket{}, storage.SealTicket{},
storage.SealSeed{}, storage.SealSeed{},

5
go.mod
View File

@ -16,6 +16,7 @@ require (
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42
github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.1
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5
@ -29,7 +30,7 @@ require (
github.com/ipfs/go-bitswap v0.1.8 github.com/ipfs/go-bitswap v0.1.8
github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
github.com/ipfs/go-car v0.0.2 github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1
github.com/ipfs/go-cid v0.0.4 github.com/ipfs/go-cid v0.0.4
github.com/ipfs/go-datastore v0.3.1 github.com/ipfs/go-datastore v0.3.1
github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521
@ -38,7 +39,6 @@ require (
github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-graphsync v0.0.4
github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1
github.com/ipfs/go-ipfs-blockstore v0.1.1 github.com/ipfs/go-ipfs-blockstore v0.1.1
github.com/ipfs/go-ipfs-blocksutil v0.0.1
github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.1
github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1
@ -52,7 +52,6 @@ require (
github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-merkledag v0.2.4
github.com/ipfs/go-path v0.0.7 github.com/ipfs/go-path v0.0.7
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785
github.com/libp2p/go-libp2p v0.4.2 github.com/libp2p/go-libp2p v0.4.2
github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-circuit v0.1.4
github.com/libp2p/go-libp2p-connmgr v0.1.0 github.com/libp2p/go-libp2p-connmgr v0.1.0

8
go.sum
View File

@ -111,6 +111,8 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY=
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww=
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ=
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
@ -120,6 +122,7 @@ github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyC
github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -210,8 +213,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0=
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ= github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ=
github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ= github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk=
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@ -696,6 +699,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

20
retrievaladapter/client.go → markets/retrievaladapter/client.go
View File

@ -3,9 +3,13 @@ package retrievaladapter
import ( import (
"context" "context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/lotus/markets/utils"
payapi "github.com/filecoin-project/lotus/node/impl/paych" payapi "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/paych"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
) )
type retrievalClientNode struct { type retrievalClientNode struct {
@ -21,21 +25,25 @@ func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retriev
// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist // GetOrCreatePaymentChannel sets up a new payment channel if one does not exist
// between a client and a miner and insures the client has the given amount of funds available in the channel // between a client and a miner and insures the client has the given amount of funds available in the channel
func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress retrievalmarket.Address, minerAddress retrievalmarket.Address, clientFundsAvailable retrievalmarket.BigInt) (retrievalmarket.Address, error) { func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) {
paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, clientFundsAvailable) paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable))
return paych, err return paych, err
} }
// Allocate late creates a lane within a payment channel so that calls to // Allocate late creates a lane within a payment channel so that calls to
// CreatePaymentVoucher will automatically make vouchers only for the difference // CreatePaymentVoucher will automatically make vouchers only for the difference
// in total // in total
func (rcn *retrievalClientNode) AllocateLane(paymentChannel retrievalmarket.Address) (uint64, error) { func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) {
return rcn.pmgr.AllocateLane(paymentChannel) return rcn.pmgr.AllocateLane(paymentChannel)
} }
// CreatePaymentVoucher creates a new payment voucher in the given lane for a // CreatePaymentVoucher creates a new payment voucher in the given lane for a
// given payment channel so that all the payment vouchers in the lane add up // given payment channel so that all the payment vouchers in the lane add up
// to the given amount (so the payment voucher will be for the difference) // to the given amount (so the payment voucher will be for the difference)
func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel retrievalmarket.Address, amount retrievalmarket.BigInt, lane uint64) (*retrievalmarket.SignedVoucher, error) { func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) {
return rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, amount, lane) voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane)
if err != nil {
return nil, err
}
return utils.ToSharedSignedVoucher(voucher)
} }

48
markets/retrievaladapter/provider.go Normal file
View File

@ -0,0 +1,48 @@
package retrievaladapter
import (
"context"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
type retrievalProviderNode struct {
sectorBlocks *sectorblocks.SectorBlocks
full api.FullNode
}
// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the
// Lotus Node
func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode {
return &retrievalProviderNode{sectorBlocks, full}
}
func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) {
asCid, err := cid.Cast(pieceCid)
if err != nil {
return 0, err
}
return rpn.sectorBlocks.GetSize(asCid)
}
func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore {
return rpn.sectorBlocks.SealedBlockstore(approveUnseal)
}
func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) {
localVoucher, err := utils.FromSharedSignedVoucher(voucher)
if err != nil {
return retrievaltoken.FromInt(0), err
}
added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount))
return utils.ToSharedTokenAmount(added), err
}

57
storagemarketadapter/client_adapter.go → markets/storageadapter/client.go
View File

@ -1,4 +1,4 @@
package storagemarketadapter package storageadapter
// this file implements storagemarket.StorageClientNode // this file implements storagemarket.StorageClientNode
@ -9,7 +9,10 @@ import (
"golang.org/x/xerrors" "golang.org/x/xerrors"
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util" cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors"
@ -18,8 +21,8 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/storagemarket"
) )
type ClientNodeAdapter struct { type ClientNodeAdapter struct {
@ -79,13 +82,8 @@ func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storag
if err != nil { if err != nil {
return nil, err return nil, err
} }
storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId)
out = append(out, &storagemarket.StorageProviderInfo{ out = append(out, &storageProviderInfo)
Address: addr,
Worker: workerAddr,
SectorSize: sectorSize,
PeerID: peerId,
})
} }
return out, nil return out, nil
@ -97,11 +95,12 @@ func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Ad
return nil, err return nil, err
} }
var out []actors.OnChainDeal var out []storagemarket.StorageDeal
for _, deal := range allDeals { for _, deal := range allDeals {
if deal.Client == addr { storageDeal := utils.FromOnChainDeal(deal)
out = append(out, deal) if storageDeal.Client == addr {
out = append(out, storageDeal)
} }
} }
@ -113,12 +112,12 @@ func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarke
} }
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
// (Provider Node API) // (Provider Node API)
smsg, err := n.MpoolPushMessage(ctx, &types.Message{ smsg, err := n.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress, To: actors.StorageMarketAddress,
From: addr, From: addr,
Value: types.BigInt(amount), Value: utils.FromSharedTokenAmount(amount),
GasPrice: types.NewInt(0), GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000), GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance, Method: actors.SMAMethods.AddBalance,
@ -139,8 +138,8 @@ func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address,
return nil return nil
} }
func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
return n.fm.EnsureAvailable(ctx, addr, types.BigInt(amount)) return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount))
} }
func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) { func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
@ -149,7 +148,7 @@ func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address
return storagemarket.Balance{}, err return storagemarket.Balance{}, err
} }
return bal, nil return utils.ToSharedBalance(bal), nil
} }
// ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal // ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal
@ -301,15 +300,29 @@ func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider
return nil return nil
} }
func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error { func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error {
return api.SignWith(ctx, n.Wallet.Sign, signer, proposal) localProposal, err := utils.FromSharedStorageDealProposal(proposal)
if err != nil {
return err
}
err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal)
if err != nil {
return err
}
signature, err := utils.ToSharedSignature(localProposal.ProposerSignature)
if err != nil {
return err
}
proposal.ProposerSignature = signature
return nil
} }
func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) { func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) {
return n.Wallet.GetDefault() addr, err := n.Wallet.GetDefault()
return addr, err
} }
func (n *ClientNodeAdapter) ValidateAskSignature(ask *types.SignedStorageAsk) error { func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error {
tss := n.cs.GetHeaviestTipSet().ParentState() tss := n.cs.GetHeaviestTipSet().ParentState()
w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner) w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner)

43
storagemarketadapter/provider_adapter.go → markets/storageadapter/provider.go
View File

@ -1,4 +1,4 @@
package storagemarketadapter package storageadapter
// this file implements storagemarket.StorageProviderNode // this file implements storagemarket.StorageProviderNode
@ -12,13 +12,16 @@ import (
"golang.org/x/xerrors" "golang.org/x/xerrors"
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/padreader" "github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/filecoin-project/lotus/storage/sectorblocks"
"github.com/filecoin-project/lotus/storagemarket"
) )
var log = logging.Logger("provideradapter") var log = logging.Logger("provideradapter")
@ -48,8 +51,12 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark
return 0, cid.Undef, err return 0, cid.Undef, err
} }
localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal)
if err != nil {
return 0, cid.Undef, err
}
params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
Deals: []actors.StorageDealProposal{deal.Proposal}, Deals: []actors.StorageDealProposal{*localProposal},
}) })
if err != nil { if err != nil {
@ -124,17 +131,18 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema
return sectorID, nil return sectorID, nil
} }
func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]actors.OnChainDeal, error) { func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) {
allDeals, err := n.StateMarketDeals(ctx, nil) allDeals, err := n.StateMarketDeals(ctx, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var out []actors.OnChainDeal var out []storagemarket.StorageDeal
for _, deal := range allDeals { for _, deal := range allDeals {
if deal.Provider == addr { sharedDeal := utils.FromOnChainDeal(deal)
out = append(out, deal) if sharedDeal.Provider == addr {
out = append(out, sharedDeal)
} }
} }
@ -142,15 +150,20 @@ func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr addres
} }
func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) { func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) {
return n.StateMinerWorker(ctx, miner, nil) addr, err := n.StateMinerWorker(ctx, miner, nil)
return addr, err
} }
func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error) { func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) {
return n.WalletSign(ctx, signer, b) localSignature, err := n.WalletSign(ctx, signer, b)
if err != nil {
return nil, err
}
return utils.ToSharedSignature(localSignature)
} }
func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt storagemarket.TokenAmount) error { func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error {
return n.MarketEnsureAvailable(ctx, addr, types.BigInt(amt)) return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt))
} }
func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) { func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) {
@ -158,12 +171,12 @@ func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemar
} }
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients. // Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount storagemarket.TokenAmount) error { func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
// (Provider Node API) // (Provider Node API)
smsg, err := n.MpoolPushMessage(ctx, &types.Message{ smsg, err := n.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress, To: actors.StorageMarketAddress,
From: addr, From: addr,
Value: types.BigInt(amount), Value: utils.FromSharedTokenAmount(amount),
GasPrice: types.NewInt(0), GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000), GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance, Method: actors.SMAMethods.AddBalance,
@ -190,7 +203,7 @@ func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Addre
return storagemarket.Balance{}, err return storagemarket.Balance{}, err
} }
return bal, nil return utils.ToSharedBalance(bal), nil
} }
var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{} var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{}

174
markets/utils/converters.go Normal file
View File

@ -0,0 +1,174 @@
package utils
import (
"bytes"
peer "github.com/libp2p/go-libp2p-peer"
"github.com/filecoin-project/go-address"
sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
)
func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt {
return types.BigInt{Int: in.Int}
}
func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount {
return sharedamount.TokenAmount{Int: in.Int}
}
func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.SignedVoucher
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.SignedVoucher
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.Signature
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.Signature
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.SignedStorageAsk
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.SignedStorageAsk
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo {
return storagemarket.StorageProviderInfo{
Address: address,
Worker: miner,
SectorSize: sectorSize,
PeerID: peer,
}
}
func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal {
return storagemarket.StorageDeal{
PieceRef: deal.PieceRef,
PieceSize: deal.PieceSize,
Client: deal.Client,
Provider: deal.Provider,
StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch),
StorageCollateral: ToSharedTokenAmount(deal.StorageCollateral),
ActivationEpoch: deal.ActivationEpoch,
}
}
func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal {
return actors.OnChainDeal{
PieceRef: deal.PieceRef,
PieceSize: deal.PieceSize,
Client: deal.Client,
Provider: deal.Provider,
StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch),
StorageCollateral: FromSharedTokenAmount(deal.StorageCollateral),
ActivationEpoch: deal.ActivationEpoch,
}
}
func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance {
return storagemarket.Balance{
Locked: ToSharedTokenAmount(balance.Locked),
Available: ToSharedTokenAmount(balance.Available),
}
}
func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) {
var encoded bytes.Buffer
err := proposal.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out storagemarket.StorageDealProposal
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) {
var encoded bytes.Buffer
err := proposal.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out actors.StorageDealProposal
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}

24
node/builder.go
View File

@ -19,10 +19,13 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"golang.org/x/xerrors" "golang.org/x/xerrors"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/market" "github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/messagepool"
@ -31,6 +34,7 @@ import (
"github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/hello"
@ -43,12 +47,8 @@ import (
"github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/peermgr"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/filecoin-project/lotus/storage/sectorblocks"
"github.com/filecoin-project/lotus/storagemarket"
"github.com/filecoin-project/lotus/storagemarketadapter"
) )
// special is a type used to give keys to modules which // special is a type used to give keys to modules which
@ -222,15 +222,15 @@ func Online() Option {
Override(RunPeerMgrKey, modules.RunPeerMgr), Override(RunPeerMgrKey, modules.RunPeerMgr),
Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks), Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
Override(new(*discovery.Local), discovery.NewLocal), Override(new(*discovery.Local), modules.NewLocalDiscovery),
Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver), Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver),
Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient), Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient),
Override(new(dtypes.ClientDealStore), modules.NewClientDealStore), Override(new(dtypes.ClientDealStore), modules.NewClientDealStore),
Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer), Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer),
Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator), Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator),
Override(new(storagemarket.StorageClient), deals.NewClient), Override(new(storagemarket.StorageClient), modules.StorageClient),
Override(new(storagemarket.StorageClientNode), storagemarketadapter.NewClientNodeAdapter), Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
Override(RegisterClientValidatorKey, modules.RegisterClientValidator), Override(RegisterClientValidatorKey, modules.RegisterClientValidator),
Override(RunDealClientKey, modules.RunDealClient), Override(RunDealClientKey, modules.RunDealClient),
@ -252,9 +252,9 @@ func Online() Option {
Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider),
Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore), Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer), Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer),
Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator), Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator),
Override(new(storagemarket.StorageProvider), deals.NewProvider), Override(new(storagemarket.StorageProvider), modules.StorageProvider),
Override(new(storagemarket.StorageProviderNode), storagemarketadapter.NewProviderNodeAdapter), Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter),
Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator), Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator),
Override(HandleRetrievalKey, modules.HandleRetrieval), Override(HandleRetrievalKey, modules.HandleRetrieval),
Override(GetParamsKey, modules.GetParams), Override(GetParamsKey, modules.GetParams),

40
node/impl/client/client.go
View File

@ -25,15 +25,17 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storagemarket"
) )
type API struct { type API struct {
@ -72,11 +74,7 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad
if err != nil { if err != nil {
return nil, xerrors.Errorf("failed getting miner worker: %w", err) return nil, xerrors.Errorf("failed getting miner worker: %w", err)
} }
providerInfo := storagemarket.StorageProviderInfo{ providerInfo := utils.NewStorageProviderInfo(miner, mw, 0, pid)
Address: miner,
Worker: mw,
PeerID: pid,
}
result, err := a.SMDealClient.ProposeStorageDeal( result, err := a.SMDealClient.ProposeStorageDeal(
ctx, ctx,
addr, addr,
@ -84,8 +82,8 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad
data, data,
storagemarket.Epoch(math.MaxUint64), storagemarket.Epoch(math.MaxUint64),
storagemarket.Epoch(blocksDuration), storagemarket.Epoch(blocksDuration),
storagemarket.TokenAmount(epochPrice), utils.ToSharedTokenAmount(epochPrice),
storagemarket.TokenAmount(storagemarket.EmptyInt)) tokenamount.Empty)
if err != nil { if err != nil {
return nil, xerrors.Errorf("failed to start deal: %w", err) return nil, xerrors.Errorf("failed to start deal: %w", err)
@ -110,7 +108,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
PieceRef: v.Proposal.PieceRef, PieceRef: v.Proposal.PieceRef,
Size: v.Proposal.PieceSize, Size: v.Proposal.PieceSize,
PricePerEpoch: v.Proposal.StoragePricePerEpoch, PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch),
Duration: v.Proposal.Duration, Duration: v.Proposal.Duration,
} }
} }
@ -130,7 +128,7 @@ func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo,
Provider: v.Proposal.Provider, Provider: v.Proposal.Provider,
PieceRef: v.Proposal.PieceRef, PieceRef: v.Proposal.PieceRef,
Size: v.Proposal.PieceSize, Size: v.Proposal.PieceSize,
PricePerEpoch: v.Proposal.StoragePricePerEpoch, PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch),
Duration: v.Proposal.Duration, Duration: v.Proposal.Duration,
}, nil }, nil
} }
@ -164,7 +162,7 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe
out[k] = api.QueryOffer{ out[k] = api.QueryOffer{
Root: root, Root: root,
Size: queryResponse.Size, Size: queryResponse.Size,
MinPrice: queryResponse.PieceRetrievalPrice(), MinPrice: utils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()),
Miner: p.Address, // TODO: check Miner: p.Address, // TODO: check
MinerPeerID: p.ID, MinerPeerID: p.ID,
} }
@ -292,9 +290,13 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
}) })
a.Retrieval.Retrieve( a.Retrieval.Retrieve(
ctx, order.Root.Bytes(), retrievalmarket.Params{ ctx,
PricePerByte: types.BigDiv(order.Total, types.NewInt(order.Size)), order.Root.Bytes(),
}, order.Total, order.MinerPeerID, order.Client, order.Miner) retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0),
utils.ToSharedTokenAmount(order.Total),
order.MinerPeerID,
order.Client,
order.Miner)
select { select {
case <-ctx.Done(): case <-ctx.Done():
return xerrors.New("Retrieval Timed Out") return xerrors.New("Retrieval Timed Out")
@ -318,6 +320,10 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
} }
func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) { func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) {
info := storagemarket.StorageProviderInfo{Address: miner, PeerID: p} info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p)
return a.SMDealClient.GetAsk(ctx, info) signedAsk, err := a.SMDealClient.GetAsk(ctx, info)
if err != nil {
return nil, err
}
return utils.FromSignedStorageAsk(signedAsk)
} }

23
node/modules/client.go
View File

@ -2,15 +2,21 @@ package modules
import ( import (
"context" "context"
"github.com/filecoin-project/lotus/retrievaladapter"
"path/filepath" "path/filepath"
"reflect" "reflect"
"github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
"github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/paych" "github.com/filecoin-project/lotus/paych"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl"
"github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-bitswap/network"
graphsync "github.com/ipfs/go-graphsync/impl" graphsync "github.com/ipfs/go-graphsync/impl"
@ -29,7 +35,8 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"github.com/filecoin-project/go-data-transfer/impl/graphsync" "github.com/filecoin-project/go-data-transfer/impl/graphsync"
"github.com/filecoin-project/lotus/chain/deals" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
payapi "github.com/filecoin-project/lotus/node/impl/paych" payapi "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/node/repo"
@ -103,6 +110,14 @@ func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.Client
return gs return gs
} }
func NewClientRequestValidator(deals dtypes.ClientDealStore) *storageimpl.ClientRequestValidator {
return storageimpl.NewClientRequestValidator(deals)
}
func StorageClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) storagemarket.StorageClient {
return storageimpl.NewClient(h, dag, dataTransfer, discovery, deals, scn)
}
// RetrievalClient creates a new retrieval client attached to the client blockstore // RetrievalClient creates a new retrieval client attached to the client blockstore
func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient { func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient {
adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi) adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi)

11
node/modules/services.go
View File

@ -8,16 +8,17 @@ import (
pubsub "github.com/libp2p/go-libp2p-pubsub" pubsub "github.com/libp2p/go-libp2p-pubsub"
"go.uber.org/fx" "go.uber.org/fx"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain" "github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/blocksync" "github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/messagepool" "github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/sub" "github.com/filecoin-project/lotus/chain/sub"
"github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/hello"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/peermgr" "github.com/filecoin-project/lotus/peermgr"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storagemarket"
) )
func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) { func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) {
@ -81,6 +82,10 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c storagemarket.Sto
}) })
} }
func NewLocalDiscovery(ds dtypes.MetadataDS) *discovery.Local {
return discovery.NewLocal(ds)
}
func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver { func RetrievalResolver(l *discovery.Local) retrievalmarket.PeerResolver {
return discovery.Multi(l) return discovery.Multi(l)
} }

24
node/modules/storageminer.go
View File

@ -25,22 +25,24 @@ import (
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync" dtgraphsync "github.com/filecoin-project/go-data-transfer/impl/graphsync"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-statestore" "github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/node/repo"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl" "github.com/filecoin-project/lotus/markets/retrievaladapter"
"github.com/filecoin-project/lotus/retrievaladapter"
"github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks" "github.com/filecoin-project/lotus/storage/sectorblocks"
"github.com/filecoin-project/lotus/storagemarket"
) )
func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
@ -263,8 +265,16 @@ func SealTicketGen(api api.FullNode) storage.TicketFn {
} }
} }
func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *storageimpl.ProviderRequestValidator {
return storageimpl.NewProviderRequestValidator(deals)
}
func StorageProvider(ds dtypes.MetadataDS, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, spn storagemarket.StorageProviderNode) (storagemarket.StorageProvider, error) {
return storageimpl.NewProvider(ds, dag, dataTransfer, spn)
}
// RetrievalProvider creates a new retrieval provider attached to the provider blockstore // RetrievalProvider creates a new retrieval provider attached to the provider blockstore
func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider { func RetrievalProvider(sblks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProvider {
adapter := retrievaladapter.NewRetrievalProviderNode(full) adapter := retrievaladapter.NewRetrievalProviderNode(sblks, full)
return retrievalimpl.NewProvider(sblks, adapter) return retrievalimpl.NewProvider(adapter)
} }

20
retrieval/cbor-gen/main.go
View File

@ -1,20 +0,0 @@
package main
import (
"fmt"
"os"
retrievalimpl "github.com/filecoin-project/lotus/retrieval/impl"
)
// main func has ONE JOB
func main() {
fmt.Print("Generating Cbor Marshal/Unmarshal...")
if err := retrievalimpl.RunCborGen(); err != nil {
fmt.Println("Failed: ")
fmt.Println(err)
os.Exit(1)
}
fmt.Println("Done.")
}

15
retrieval/discovery/discovery.go
View File

@ -1,15 +0,0 @@
package discovery
import (
cbor "github.com/ipfs/go-ipld-cbor"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
)
func init() {
cbor.RegisterCborType(retrievalmarket.RetrievalPeer{})
}
func Multi(r retrievalmarket.PeerResolver) retrievalmarket.PeerResolver { // TODO: actually support multiple mechanisms
return r
}

54
retrieval/discovery/local.go
View File

@ -1,54 +0,0 @@
package discovery
import (
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
dshelp "github.com/ipfs/go-ipfs-ds-help"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/node/modules/dtypes"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
)
var log = logging.Logger("ret-discovery")
type Local struct {
ds datastore.Datastore
}
func NewLocal(ds dtypes.MetadataDS) *Local {
return &Local{ds: namespace.Wrap(ds, datastore.NewKey("/deals/local"))}
}
func (l *Local) AddPeer(cid cid.Cid, peer retrievalmarket.RetrievalPeer) error {
// TODO: allow multiple peers here
// (implement an util for tracking map[thing][]otherThing, use in sectorBlockstore too)
log.Warn("Tracking multiple retrieval peers not implemented")
entry, err := cbor.DumpObject(peer)
if err != nil {
return err
}
return l.ds.Put(dshelp.CidToDsKey(cid), entry)
}
func (l *Local) GetPeers(data cid.Cid) ([]retrievalmarket.RetrievalPeer, error) {
entry, err := l.ds.Get(dshelp.CidToDsKey(data))
if err == datastore.ErrNotFound {
return []retrievalmarket.RetrievalPeer{}, nil
}
if err != nil {
return nil, err
}
var peer retrievalmarket.RetrievalPeer
if err := cbor.DecodeInto(entry, &peer); err != nil {
return nil, err
}
return []retrievalmarket.RetrievalPeer{peer}, nil
}
var _ retrievalmarket.PeerResolver = &Local{}

476
retrieval/impl/cbor_gen.go
View File

@ -1,476 +0,0 @@
package retrievalimpl
import (
"fmt"
"io"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
var _ = xerrors.Errorf
func (t *RetParams) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct)
if err := t.Unixfs0.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *RetParams) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Unixfs0 (retrievalimpl.Unixfs0Offer) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Unixfs0 = new(Unixfs0Offer)
if err := t.Unixfs0.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *OldQuery) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Piece (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Piece); err != nil {
return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
}
return nil
}
func (t *OldQuery) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Piece (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
}
t.Piece = c
}
return nil
}
func (t *OldQueryResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.Status (retrievalimpl.OldQueryResponseStatus) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil {
return err
}
// t.Size (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
return err
}
// t.MinPrice (types.BigInt) (struct)
if err := t.MinPrice.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *OldQueryResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Status (retrievalimpl.OldQueryResponseStatus) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = OldQueryResponseStatus(extra)
// t.Size (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = uint64(extra)
// t.MinPrice (types.BigInt) (struct)
{
if err := t.MinPrice.UnmarshalCBOR(br); err != nil {
return err
}
}
return nil
}
func (t *Unixfs0Offer) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Offset (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil {
return err
}
// t.Size (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
return err
}
return nil
}
func (t *Unixfs0Offer) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Offset (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Offset = uint64(extra)
// t.Size (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = uint64(extra)
return nil
}
func (t *OldDealProposal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.Payment (api.PaymentInfo) (struct)
if err := t.Payment.MarshalCBOR(w); err != nil {
return err
}
// t.Ref (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Ref); err != nil {
return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
}
// t.Params (retrievalimpl.RetParams) (struct)
if err := t.Params.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *OldDealProposal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Payment (api.PaymentInfo) (struct)
{
if err := t.Payment.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Ref (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
}
t.Ref = c
}
// t.Params (retrievalimpl.RetParams) (struct)
{
if err := t.Params.UnmarshalCBOR(br); err != nil {
return err
}
}
return nil
}
func (t *OldDealResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Status (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil {
return err
}
// t.Message (string) (string)
if len(t.Message) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Message was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Message)); err != nil {
return err
}
return nil
}
func (t *OldDealResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Status (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Status = uint64(extra)
// t.Message (string) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Message = string(sval)
}
return nil
}
func (t *Block) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Prefix ([]uint8) (slice)
if len(t.Prefix) > cbg.ByteArrayMaxLen {
return xerrors.Errorf("Byte array in field t.Prefix was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Prefix)))); err != nil {
return err
}
if _, err := w.Write(t.Prefix); err != nil {
return err
}
// t.Data ([]uint8) (slice)
if len(t.Data) > cbg.ByteArrayMaxLen {
return xerrors.Errorf("Byte array in field t.Data was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Data)))); err != nil {
return err
}
if _, err := w.Write(t.Data); err != nil {
return err
}
return nil
}
func (t *Block) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Prefix ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > cbg.ByteArrayMaxLen {
return fmt.Errorf("t.Prefix: byte array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Prefix = make([]byte, extra)
if _, err := io.ReadFull(br, t.Prefix); err != nil {
return err
}
// t.Data ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > cbg.ByteArrayMaxLen {
return fmt.Errorf("t.Data: byte array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Data = make([]byte, extra)
if _, err := io.ReadFull(br, t.Data); err != nil {
return err
}
return nil
}

389
retrieval/impl/client.go
View File

@ -1,389 +0,0 @@
package retrievalimpl
import (
"context"
"reflect"
"sync"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
)
var log = logging.Logger("retrieval")
type client struct {
h host.Host
bs blockstore.Blockstore
node retrievalmarket.RetrievalClientNode
// The parameters should be replaced by RetrievalClientNode
nextDealLk sync.Mutex
nextDealID retrievalmarket.DealID
subscribersLk sync.RWMutex
subscribers []retrievalmarket.ClientSubscriber
}
// NewClient creates a new retrieval client
func NewClient(h host.Host, bs blockstore.Blockstore, node retrievalmarket.RetrievalClientNode) retrievalmarket.RetrievalClient {
return &client{h: h, bs: bs, node: node}
}
// V0
// TODO: Implement for retrieval provider V0 epic
// https://github.com/filecoin-project/go-retrieval-market-project/issues/12
func (c *client) FindProviders(pieceCID []byte) []retrievalmarket.RetrievalPeer {
panic("not implemented")
}
// TODO: Update to match spec for V0 epic
// https://github.com/filecoin-project/go-retrieval-market-project/issues/8
func (c *client) Query(ctx context.Context, p retrievalmarket.RetrievalPeer, pieceCID []byte, params retrievalmarket.QueryParams) (retrievalmarket.QueryResponse, error) {
cid, err := cid.Cast(pieceCID)
if err != nil {
log.Warn(err)
return retrievalmarket.QueryResponseUndefined, err
}
s, err := c.h.NewStream(ctx, p.ID, retrievalmarket.QueryProtocolID)
if err != nil {
log.Warn(err)
return retrievalmarket.QueryResponseUndefined, err
}
defer s.Close()
err = cborutil.WriteCborRPC(s, &OldQuery{
Piece: cid,
})
if err != nil {
log.Warn(err)
return retrievalmarket.QueryResponseUndefined, err
}
var oldResp OldQueryResponse
if err := oldResp.UnmarshalCBOR(s); err != nil {
log.Warn(err)
return retrievalmarket.QueryResponseUndefined, err
}
resp := retrievalmarket.QueryResponse{
Status: retrievalmarket.QueryResponseStatus(oldResp.Status),
Size: oldResp.Size,
MinPricePerByte: types.BigDiv(oldResp.MinPrice, types.NewInt(oldResp.Size)),
}
return resp, nil
}
// TODO: Update to match spec for V0 Epic:
// https://github.com/filecoin-project/go-retrieval-market-project/issues/9
func (c *client) Retrieve(ctx context.Context, pieceCID []byte, params retrievalmarket.Params, totalFunds types.BigInt, miner peer.ID, clientWallet retrievalmarket.Address, minerWallet retrievalmarket.Address) retrievalmarket.DealID {
/* The implementation of this function is just wrapper for the old code which retrieves UnixFS pieces
-- it will be replaced when we do the V0 implementation of the module */
c.nextDealLk.Lock()
c.nextDealID++
dealID := c.nextDealID
c.nextDealLk.Unlock()
dealState := retrievalmarket.ClientDealState{
DealProposal: retrievalmarket.DealProposal{
PieceCID: pieceCID,
ID: dealID,
Params: params,
},
Status: retrievalmarket.DealStatusFailed,
Sender: miner,
}
go func() {
evt := retrievalmarket.ClientEventError
converted, err := cid.Cast(pieceCID)
if err == nil {
err = c.retrieveUnixfs(ctx, converted, types.BigDiv(totalFunds, params.PricePerByte).Uint64(), totalFunds, miner, clientWallet, minerWallet)
if err == nil {
evt = retrievalmarket.ClientEventComplete
dealState.Status = retrievalmarket.DealStatusCompleted
}
}
c.notifySubscribers(evt, dealState)
}()
return dealID
}
// unsubscribeAt returns a function that removes an item from the subscribers list by comparing
// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order.
// Subsequent, repeated calls to the func with the same Subscriber are a no-op.
func (c *client) unsubscribeAt(sub retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe {
return func() {
c.subscribersLk.Lock()
defer c.subscribersLk.Unlock()
curLen := len(c.subscribers)
for i, el := range c.subscribers {
if reflect.ValueOf(sub) == reflect.ValueOf(el) {
c.subscribers[i] = c.subscribers[curLen-1]
c.subscribers = c.subscribers[:curLen-1]
return
}
}
}
}
func (c *client) notifySubscribers(evt retrievalmarket.ClientEvent, ds retrievalmarket.ClientDealState) {
c.subscribersLk.RLock()
defer c.subscribersLk.RUnlock()
for _, cb := range c.subscribers {
cb(evt, ds)
}
}
func (c *client) SubscribeToEvents(subscriber retrievalmarket.ClientSubscriber) retrievalmarket.Unsubscribe {
c.subscribersLk.Lock()
c.subscribers = append(c.subscribers, subscriber)
c.subscribersLk.Unlock()
return c.unsubscribeAt(subscriber)
}
// V1
func (c *client) AddMoreFunds(id retrievalmarket.DealID, amount types.BigInt) error {
panic("not implemented")
}
func (c *client) CancelDeal(id retrievalmarket.DealID) error {
panic("not implemented")
}
func (c *client) RetrievalStatus(id retrievalmarket.DealID) {
panic("not implemented")
}
func (c *client) ListDeals() map[retrievalmarket.DealID]retrievalmarket.ClientDealState {
panic("not implemented")
}
type clientStream struct {
node retrievalmarket.RetrievalClientNode
stream network.Stream
peeker cbg.BytePeeker
root cid.Cid
size types.BigInt
offset uint64
paych retrievalmarket.Address
lane uint64
total types.BigInt
transferred types.BigInt
windowSize uint64 // how much we "trust" the peer
verifier BlockVerifier
bs blockstore.Blockstore
}
/* This is the old retrieval code that is NOT spec compliant */
// C > S
//
// Offset MUST be aligned on chunking boundaries, size is rounded up to leaf size
//
// > DealProposal{Mode: Unixfs0, RootCid, Offset, Size, Payment(nil if free)}
// < Resp{Accept}
// < ..(Intermediate Block)
// < ..Blocks
// < ..(Intermediate Block)
// < ..Blocks
// > DealProposal(...)
// < ...
func (c *client) retrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr retrievalmarket.Address) error {
s, err := c.h.NewStream(ctx, miner, retrievalmarket.ProtocolID)
if err != nil {
return err
}
defer s.Close()
initialOffset := uint64(0) // TODO: Check how much data we have locally
// TODO: Support in handler
// TODO: Allow client to specify this
paych, err := c.node.GetOrCreatePaymentChannel(ctx, client, minerAddr, total)
if err != nil {
return xerrors.Errorf("getting payment channel: %w", err)
}
lane, err := c.node.AllocateLane(paych)
if err != nil {
return xerrors.Errorf("allocating payment lane: %w", err)
}
cst := clientStream{
node: c.node,
stream: s,
peeker: cbg.GetPeeker(s),
root: root,
size: types.NewInt(size),
offset: initialOffset,
paych: paych,
lane: lane,
total: total,
transferred: types.NewInt(0),
windowSize: build.UnixfsChunkSize,
verifier: &UnixFs0Verifier{Root: root},
bs: c.bs,
}
for cst.offset != size+initialOffset {
toFetch := cst.windowSize
if toFetch+cst.offset > size {
toFetch = size - cst.offset
}
log.Infof("Retrieve %dB @%d", toFetch, cst.offset)
err := cst.doOneExchange(ctx, toFetch)
if err != nil {
return xerrors.Errorf("retrieval exchange: %w", err)
}
cst.offset += toFetch
}
log.Info("RETRIEVE SUCCESSFUL")
return nil
}
func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64) error {
payAmount := types.BigDiv(types.BigMul(cst.total, types.NewInt(toFetch)), cst.size)
payment, err := cst.setupPayment(ctx, payAmount)
if err != nil {
return xerrors.Errorf("setting up retrieval payment: %w", err)
}
deal := &OldDealProposal{
Payment: payment,
Ref: cst.root,
Params: RetParams{
Unixfs0: &Unixfs0Offer{
Offset: cst.offset,
Size: toFetch,
},
},
}
if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil {
return err
}
var resp OldDealResponse
if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil {
log.Error(err)
return err
}
if resp.Status != Accepted {
cst.windowSize = build.UnixfsChunkSize
// TODO: apply some 'penalty' to miner 'reputation' (needs to be the same in both cases)
if resp.Status == Error {
return xerrors.Errorf("storage deal error: %s", resp.Message)
}
if resp.Status == Rejected {
return xerrors.Errorf("storage deal rejected: %s", resp.Message)
}
return xerrors.New("storage deal response had no Accepted section")
}
log.Info("Retrieval accepted, fetching blocks")
return cst.fetchBlocks(toFetch)
// TODO: maybe increase miner window size after success
}
func (cst *clientStream) fetchBlocks(toFetch uint64) error {
blocksToFetch := (toFetch + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize
for i := uint64(0); i < blocksToFetch; {
log.Infof("block %d of %d", i+1, blocksToFetch)
var block Block
if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil {
return xerrors.Errorf("reading fetchBlock response: %w", err)
}
dataBlocks, err := cst.consumeBlockMessage(block)
if err != nil {
return xerrors.Errorf("consuming retrieved blocks: %w", err)
}
i += dataBlocks
}
return nil
}
func (cst *clientStream) consumeBlockMessage(block Block) (uint64, error) {
prefix, err := cid.PrefixFromBytes(block.Prefix)
if err != nil {
return 0, err
}
cid, err := prefix.Sum(block.Data)
blk, err := blocks.NewBlockWithCid(block.Data, cid)
if err != nil {
return 0, err
}
internal, err := cst.verifier.Verify(context.TODO(), blk)
if err != nil {
log.Warnf("block verify failed: %s", err)
return 0, err
}
// TODO: Smarter out, maybe add to filestore automagically
// (Also, persist intermediate nodes)
err = cst.bs.Put(blk)
if err != nil {
log.Warnf("block write failed: %s", err)
return 0, err
}
if internal {
return 0, nil
}
return 1, nil
}
func (cst *clientStream) setupPayment(ctx context.Context, toSend types.BigInt) (api.PaymentInfo, error) {
amount := types.BigAdd(cst.transferred, toSend)
sv, err := cst.node.CreatePaymentVoucher(ctx, cst.paych, amount, cst.lane)
if err != nil {
return api.PaymentInfo{}, err
}
cst.transferred = amount
return api.PaymentInfo{
Channel: cst.paych,
ChannelMessage: nil,
Vouchers: []*types.SignedVoucher{sv},
}, nil
}

323
retrieval/impl/provider.go
View File

@ -1,323 +0,0 @@
package retrievalimpl
import (
"context"
"io"
"reflect"
"sync"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-merkledag"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/network"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
// RetrMinerAPI are the node functions needed by a retrieval provider
type RetrMinerAPI interface {
PaychVoucherAdd(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)
}
type provider struct {
// TODO: Replace with RetrievalProviderNode & FileStore for https://github.com/filecoin-project/go-retrieval-market-project/issues/9
sectorBlocks *sectorblocks.SectorBlocks
// TODO: Replace with RetrievalProviderNode for
// https://github.com/filecoin-project/go-retrieval-market-project/issues/4
node retrievalmarket.RetrievalProviderNode
pricePerByte retrievalmarket.BigInt
subscribersLk sync.RWMutex
subscribers []retrievalmarket.ProviderSubscriber
}
// NewProvider returns a new retrieval provider
func NewProvider(sblks *sectorblocks.SectorBlocks, node retrievalmarket.RetrievalProviderNode) retrievalmarket.RetrievalProvider {
return &provider{
sectorBlocks: sblks,
node: node,
pricePerByte: types.NewInt(2), // TODO: allow setting
}
}
// Start begins listening for deals on the given host
func (p *provider) Start(host host.Host) {
host.SetStreamHandler(retrievalmarket.QueryProtocolID, p.handleQueryStream)
host.SetStreamHandler(retrievalmarket.ProtocolID, p.handleDealStream)
}
// V0
// SetPricePerByte sets the price per byte a miner charges for retrievals
func (p *provider) SetPricePerByte(price retrievalmarket.BigInt) {
p.pricePerByte = price
}
// SetPaymentInterval sets the maximum number of bytes a a provider will send before
// requesting further payment, and the rate at which that value increases
// TODO: Implement for https://github.com/filecoin-project/go-retrieval-market-project/issues/7
func (p *provider) SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64) {
panic("not implemented")
}
// unsubscribeAt returns a function that removes an item from the subscribers list by comparing
// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order.
// Subsequent, repeated calls to the func with the same Subscriber are a no-op.
func (p *provider) unsubscribeAt(sub retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe {
return func() {
p.subscribersLk.Lock()
defer p.subscribersLk.Unlock()
curLen := len(p.subscribers)
for i, el := range p.subscribers {
if reflect.ValueOf(sub) == reflect.ValueOf(el) {
p.subscribers[i] = p.subscribers[curLen-1]
p.subscribers = p.subscribers[:curLen-1]
return
}
}
}
}
func (p *provider) notifySubscribers(evt retrievalmarket.ProviderEvent, ds retrievalmarket.ProviderDealState) {
p.subscribersLk.RLock()
defer p.subscribersLk.RUnlock()
for _, cb := range p.subscribers {
cb(evt, ds)
}
}
// SubscribeToEvents listens for events that happen related to client retrievals
// TODO: Implement updates as part of https://github.com/filecoin-project/go-retrieval-market-project/issues/7
func (p *provider) SubscribeToEvents(subscriber retrievalmarket.ProviderSubscriber) retrievalmarket.Unsubscribe {
p.subscribersLk.Lock()
p.subscribers = append(p.subscribers, subscriber)
p.subscribersLk.Unlock()
return p.unsubscribeAt(subscriber)
}
// V1
func (p *provider) SetPricePerUnseal(price retrievalmarket.BigInt) {
panic("not implemented")
}
func (p *provider) ListDeals() map[retrievalmarket.ProviderDealID]retrievalmarket.ProviderDealState {
panic("not implemented")
}
func writeErr(stream network.Stream, err error) {
log.Errorf("Retrieval deal error: %+v", err)
_ = cborutil.WriteCborRPC(stream, &OldDealResponse{
Status: Error,
Message: err.Error(),
})
}
// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/8
func (p *provider) handleQueryStream(stream network.Stream) {
defer stream.Close()
var query OldQuery
if err := cborutil.ReadCborRPC(stream, &query); err != nil {
writeErr(stream, err)
return
}
size, err := p.sectorBlocks.GetSize(query.Piece)
if err != nil && err != sectorblocks.ErrNotFound {
log.Errorf("Retrieval query: GetRefs: %s", err)
return
}
answer := &OldQueryResponse{
Status: Unavailable,
}
if err == nil {
answer.Status = Available
// TODO: get price, look for already unsealed ref to reduce work
answer.MinPrice = types.BigMul(types.NewInt(uint64(size)), p.pricePerByte)
answer.Size = uint64(size) // TODO: verify on intermediate
}
if err := cborutil.WriteCborRPC(stream, answer); err != nil {
log.Errorf("Retrieval query: WriteCborRPC: %s", err)
return
}
}
type handlerDeal struct {
p *provider
stream network.Stream
ufsr sectorblocks.UnixfsReader
open cid.Cid
at uint64
size uint64
}
// TODO: Update for https://github.com/filecoin-project/go-retrieval-market-project/issues/7
func (p *provider) handleDealStream(stream network.Stream) {
defer stream.Close()
hnd := &handlerDeal{
p: p,
stream: stream,
}
var err error
more := true
for more {
more, err = hnd.handleNext() // TODO: 'more' bool
if err != nil {
writeErr(stream, err)
return
}
}
}
func (hnd *handlerDeal) handleNext() (bool, error) {
var deal OldDealProposal
if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil {
if err == io.EOF { // client sent all deals
err = nil
}
return false, err
}
if deal.Params.Unixfs0 == nil {
return false, xerrors.New("unknown deal type")
}
unixfs0 := deal.Params.Unixfs0
if len(deal.Payment.Vouchers) != 1 {
return false, xerrors.Errorf("expected one signed voucher, got %d", len(deal.Payment.Vouchers))
}
expPayment := types.BigMul(hnd.p.pricePerByte, types.NewInt(deal.Params.Unixfs0.Size))
if _, err := hnd.p.node.SavePaymentVoucher(context.TODO(), deal.Payment.Channel, deal.Payment.Vouchers[0], nil, expPayment); err != nil {
return false, xerrors.Errorf("processing retrieval payment: %w", err)
}
// If the file isn't open (new deal stream), isn't the right file, or isn't
// at the right offset, (re)open it
if hnd.open != deal.Ref || hnd.at != unixfs0.Offset {
log.Infof("opening file for sending (open '%s') (@%d, want %d)", deal.Ref, hnd.at, unixfs0.Offset)
if err := hnd.openFile(deal); err != nil {
return false, err
}
}
if unixfs0.Offset+unixfs0.Size > hnd.size {
return false, xerrors.Errorf("tried to read too much %d+%d > %d", unixfs0.Offset, unixfs0.Size, hnd.size)
}
err := hnd.accept(deal)
if err != nil {
return false, err
}
return true, nil
}
func (hnd *handlerDeal) openFile(deal OldDealProposal) error {
unixfs0 := deal.Params.Unixfs0
if unixfs0.Offset != 0 {
// TODO: Implement SeekBlock (like ReadBlock) in go-unixfs
return xerrors.New("sending merkle proofs for nonzero offset not supported yet")
}
hnd.at = unixfs0.Offset
bstore := hnd.p.sectorBlocks.SealedBlockstore(func() error {
return nil // TODO: approve unsealing based on amount paid
})
ds := merkledag.NewDAGService(blockservice.New(bstore, nil))
rootNd, err := ds.Get(context.TODO(), deal.Ref)
if err != nil {
return err
}
fsr, err := unixfile.NewUnixfsFile(context.TODO(), ds, rootNd)
if err != nil {
return err
}
var ok bool
hnd.ufsr, ok = fsr.(sectorblocks.UnixfsReader)
if !ok {
return xerrors.Errorf("file %s didn't implement sectorblocks.UnixfsReader", deal.Ref)
}
isize, err := hnd.ufsr.Size()
if err != nil {
return err
}
hnd.size = uint64(isize)
hnd.open = deal.Ref
return nil
}
func (hnd *handlerDeal) accept(deal OldDealProposal) error {
unixfs0 := deal.Params.Unixfs0
resp := &OldDealResponse{
Status: Accepted,
}
if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil {
log.Errorf("Retrieval query: Write Accepted resp: %s", err)
return err
}
blocksToSend := (unixfs0.Size + build.UnixfsChunkSize - 1) / build.UnixfsChunkSize
for i := uint64(0); i < blocksToSend; {
data, offset, nd, err := hnd.ufsr.ReadBlock(context.TODO())
if err != nil {
return err
}
log.Infof("sending block for a deal: %s", nd.Cid())
if offset != unixfs0.Offset {
return xerrors.Errorf("ReadBlock on wrong offset: want %d, got %d", unixfs0.Offset, offset)
}
/*if uint64(len(data)) != deal.Unixfs0.Size { // TODO: Fix for internal nodes (and any other node too)
writeErr(stream, xerrors.Errorf("ReadBlock data with wrong size: want %d, got %d", deal.Unixfs0.Size, len(data)))
return
}*/
block := &Block{
Prefix: nd.Cid().Prefix().Bytes(),
Data: nd.RawData(),
}
if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil {
return err
}
if len(data) > 0 { // don't count internal nodes
hnd.at += uint64(len(data))
i++
}
}
return nil
}

33
retrieval/impl/run_cbor_gen.go
View File

@ -1,33 +0,0 @@
package retrievalimpl
import (
"fmt"
"os"
cborgen "github.com/whyrusleeping/cbor-gen"
)
func RunCborGen() error {
genName := "./impl/cbor_gen.go"
reName := "./impl/cbor_gen_old.go"
if err := os.Rename(genName, reName); err != nil {
return fmt.Errorf("could not rename %s to %s", genName, reName)
}
if err := cborgen.WriteTupleEncodersToFile(
genName,
"retrievalimpl",
RetParams{},
OldQuery{},
OldQueryResponse{},
Unixfs0Offer{},
OldDealProposal{},
OldDealResponse{},
Block{},
); err != nil {
return err
}
if err := os.Remove(reName); err != nil {
return err
}
return nil
}

67
retrieval/impl/types.go
View File

@ -1,67 +0,0 @@
package retrievalimpl
import (
"github.com/filecoin-project/lotus/api"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/chain/types"
)
/* These types are all the types provided by Lotus, which diverge even from
spec V0 -- prior to the "update to spec epic", we are using these types internally
and switching to spec at the boundaries of the module */
type OldQueryResponseStatus uint64
const (
Available OldQueryResponseStatus = iota
Unavailable
)
const (
Accepted = iota
Error
Rejected
Unsealing
)
type OldQuery struct {
Piece cid.Cid
// TODO: payment
}
type OldQueryResponse struct {
Status OldQueryResponseStatus
Size uint64 // TODO: spec
// TODO: unseal price (+spec)
// TODO: sectors to unseal
// TODO: address to send money for the deal?
MinPrice types.BigInt
}
type Unixfs0Offer struct {
Offset uint64
Size uint64
}
type RetParams struct {
Unixfs0 *Unixfs0Offer
}
type OldDealProposal struct {
Payment api.PaymentInfo
Ref cid.Cid
Params RetParams
}
type OldDealResponse struct {
Status uint64
Message string
}
type Block struct { // TODO: put in spec
Prefix []byte // TODO: fix cid.Prefix marshaling somehow
Data []byte
}

141
retrieval/impl/verify.go
View File

@ -1,141 +0,0 @@
package retrievalimpl
import (
"context"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipld "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
"github.com/ipfs/go-unixfs"
pb "github.com/ipfs/go-unixfs/pb"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
)
type BlockVerifier interface {
Verify(context.Context, blocks.Block) (internal bool, err error)
}
type OptimisticVerifier struct {
}
func (o *OptimisticVerifier) Verify(context.Context, blocks.Block) (bool, error) {
// It's probably fine
return false, nil
}
type UnixFs0Verifier struct {
Root cid.Cid
rootBlk blocks.Block
expect int
seen int
sub *UnixFs0Verifier
}
func (b *UnixFs0Verifier) verify(ctx context.Context, blk blocks.Block) (last bool, internal bool, err error) {
if b.sub != nil {
// TODO: check links here (iff b.sub.sub == nil)
subLast, internal, err := b.sub.verify(ctx, blk)
if err != nil {
return false, false, err
}
if subLast {
b.sub = nil
b.seen++
}
return b.seen == b.expect, internal, nil
}
if b.seen >= b.expect { // this is probably impossible
return false, false, xerrors.New("unixfs verifier: too many nodes in level")
}
links, err := b.checkInternal(blk)
if err != nil {
return false, false, err
}
if links > 0 { // TODO: check if all links are intermediate (or all aren't)
if links > build.UnixfsLinksPerLevel {
return false, false, xerrors.New("unixfs verifier: too many links in intermediate node")
}
if b.seen+1 == b.expect && links != build.UnixfsLinksPerLevel {
return false, false, xerrors.New("unixfs verifier: too few nodes in level")
}
b.sub = &UnixFs0Verifier{
Root: blk.Cid(),
rootBlk: blk,
expect: links,
}
// don't mark as seen yet
return false, true, nil
}
b.seen++
return b.seen == b.expect, false, nil
}
func (b *UnixFs0Verifier) checkInternal(blk blocks.Block) (int, error) {
nd, err := ipld.Decode(blk)
if err != nil {
log.Warnf("IPLD Decode failed: %s", err)
return 0, err
}
// TODO: check size
switch nd := nd.(type) {
case *merkledag.ProtoNode:
fsn, err := unixfs.FSNodeFromBytes(nd.Data())
if err != nil {
log.Warnf("unixfs.FSNodeFromBytes failed: %s", err)
return 0, err
}
if fsn.Type() != pb.Data_File {
return 0, xerrors.New("internal nodes must be a file")
}
if len(fsn.Data()) > 0 {
return 0, xerrors.New("internal node with data")
}
if len(nd.Links()) == 0 {
return 0, xerrors.New("internal node with no links")
}
return len(nd.Links()), nil
case *merkledag.RawNode:
return 0, nil
default:
return 0, xerrors.New("verifier: unknown node type")
}
}
func (b *UnixFs0Verifier) Verify(ctx context.Context, blk blocks.Block) (bool, error) {
// root is special
if b.rootBlk == nil {
if !b.Root.Equals(blk.Cid()) {
return false, xerrors.Errorf("unixfs verifier: root block CID didn't match: valid %s, got %s", b.Root, blk.Cid())
}
b.rootBlk = blk
links, err := b.checkInternal(blk)
if err != nil {
return false, err
}
b.expect = links
return links != 0, nil
}
_, internal, err := b.verify(ctx, blk)
return internal, err
}
var _ BlockVerifier = &OptimisticVerifier{}
var _ BlockVerifier = &UnixFs0Verifier{}

364
retrieval/types.go
View File

@ -1,364 +0,0 @@
package retrievalmarket
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
"github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
)
// type aliases
// TODO: Remove and use native types or extract for
// https://github.com/filecoin-project/go-retrieval-market-project/issues/5
// BigInt is used for token amounts in retrieval deals
type BigInt = types.BigInt
// Address is an address in the filecoin network
type Address = address.Address
// SignedVoucher is a signed payment voucher
type SignedVoucher = types.SignedVoucher
// ProtocolID is the protocol for proposing / responding to retrieval deals
const ProtocolID = "/fil/retrieval/0.0.1"
// QueryProtocolID is the protocol for querying infromation about retrieval
// deal parameters
const QueryProtocolID = "/fil/retrieval/qry/0.0.1" // TODO: spec
// Unsubscribe is a function that unsubscribes a subscriber for either the
// client or the provider
type Unsubscribe func()
// ClientDealState is the current state of a deal from the point of view
// of a retrieval client
type ClientDealState struct {
DealProposal
Status DealStatus
Sender peer.ID
TotalReceived uint64
FundsSpent BigInt
}
// ClientEvent is an event that occurs in a deal lifecycle on the client
type ClientEvent uint64
const (
// ClientEventOpen indicates a deal was initiated
ClientEventOpen ClientEvent = iota
// ClientEventFundsExpended indicates a deal has run out of funds in the payment channel
// forcing the client to add more funds to continue the deal
ClientEventFundsExpended // when totalFunds is expended
// ClientEventProgress indicates more data was received for a retrieval
ClientEventProgress
// ClientEventError indicates an error occurred during a deal
ClientEventError
// ClientEventComplete indicates a deal has completed
ClientEventComplete
)
// ClientSubscriber is a callback that is registered to listen for retrieval events
type ClientSubscriber func(event ClientEvent, state ClientDealState)
// RetrievalClient is a client interface for making retrieval deals
type RetrievalClient interface {
// V0
// Find Providers finds retrieval providers who may be storing a given piece
FindProviders(pieceCID []byte) []RetrievalPeer
// Query asks a provider for information about a piece it is storing
Query(
ctx context.Context,
p RetrievalPeer,
pieceCID []byte,
params QueryParams,
) (QueryResponse, error)
// Retrieve retrieves all or part of a piece with the given retrieval parameters
Retrieve(
ctx context.Context,
pieceCID []byte,
params Params,
totalFunds BigInt,
miner peer.ID,
clientWallet Address,
minerWallet Address,
) DealID
// SubscribeToEvents listens for events that happen related to client retrievals
SubscribeToEvents(subscriber ClientSubscriber) Unsubscribe
// V1
AddMoreFunds(id DealID, amount BigInt) error
CancelDeal(id DealID) error
RetrievalStatus(id DealID)
ListDeals() map[DealID]ClientDealState
}
// RetrievalClientNode are the node depedencies for a RetrevalClient
type RetrievalClientNode interface {
// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist
// between a client and a miner and insures the client has the given amount of funds available in the channel
GetOrCreatePaymentChannel(ctx context.Context, clientAddress Address, minerAddress Address, clientFundsAvailable BigInt) (Address, error)
// Allocate late creates a lane within a payment channel so that calls to
// CreatePaymentVoucher will automatically make vouchers only for the difference
// in total
AllocateLane(paymentChannel Address) (uint64, error)
// CreatePaymentVoucher creates a new payment voucher in the given lane for a
// given payment channel so that all the payment vouchers in the lane add up
// to the given amount (so the payment voucher will be for the difference)
CreatePaymentVoucher(ctx context.Context, paymentChannel Address, amount BigInt, lane uint64) (*SignedVoucher, error)
}
// ProviderDealState is the current state of a deal from the point of view
// of a retrieval provider
type ProviderDealState struct {
DealProposal
Status DealStatus
Receiver peer.ID
TotalSent uint64
FundsReceived BigInt
}
// ProviderEvent is an event that occurs in a deal lifecycle on the provider
type ProviderEvent uint64
const (
// ProviderEventOpen indicates a new deal was received from a client
ProviderEventOpen ProviderEvent = iota
// ProviderEventProgress indicates more data was sent to a client
ProviderEventProgress
// ProviderEventError indicates an error occurred in processing a deal for a client
ProviderEventError
// ProviderEventComplete indicates a retrieval deal was completed for a client
ProviderEventComplete
)
// ProviderDealID is a unique identifier for a deal on a provider -- it is
// a combination of DealID set by the client and the peer ID of the client
type ProviderDealID struct {
From peer.ID
ID DealID
}
// ProviderSubscriber is a callback that is registered to listen for retrieval events on a provider
type ProviderSubscriber func(event ProviderEvent, state ProviderDealState)
// RetrievalProvider is an interface by which a provider configures their
// retrieval operations and monitors deals received and process
type RetrievalProvider interface {
// Start begins listening for deals on the given host
Start(host.Host)
// V0
// SetPricePerByte sets the price per byte a miner charges for retrievals
SetPricePerByte(price BigInt)
// SetPaymentInterval sets the maximum number of bytes a a provider will send before
// requesting further payment, and the rate at which that value increases
SetPaymentInterval(paymentInterval uint64, paymentIntervalIncrease uint64)
// SubscribeToEvents listens for events that happen related to client retrievals
SubscribeToEvents(subscriber ProviderSubscriber) Unsubscribe
// V1
SetPricePerUnseal(price BigInt)
ListDeals() map[ProviderDealID]ProviderDealState
}
// RetrievalProviderNode are the node depedencies for a RetrevalProvider
type RetrievalProviderNode interface {
SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *SignedVoucher, proof []byte, expectedAmount BigInt) (BigInt, error)
}
// PeerResolver is an interface for looking up providers that may have a piece
type PeerResolver interface {
GetPeers(data cid.Cid) ([]RetrievalPeer, error) // TODO: channel
}
// RetrievalPeer is a provider address/peer.ID pair (everything needed to make
// deals for with a miner)
type RetrievalPeer struct {
Address Address
ID peer.ID // optional
}
// QueryResponseStatus indicates whether a queried piece is available
type QueryResponseStatus uint64
const (
// QueryResponseAvailable indicates a provider has a piece and is prepared to
// return it
QueryResponseAvailable QueryResponseStatus = iota
// QueryResponseUnavailable indicates a provider either does not have or cannot
// serve the queried piece to the client
QueryResponseUnavailable
)
// QueryItemStatus (V1) indicates whether the requested part of a piece (payload or selector)
// is available for retrieval
type QueryItemStatus uint64
const (
// QueryItemAvailable indicates requested part of the piece is available to be
// served
QueryItemAvailable QueryItemStatus = iota
// QueryItemUnavailable indicates the piece either does not contain the requested
// item or it cannot be served
QueryItemUnavailable
// QueryItemUnknown indicates the provider cannot determine if the given item
// is part of the requested piece (for example, if the piece is sealed and the
// miner does not maintain a payload CID index)
QueryItemUnknown
)
// QueryParams indicate what specific information about a piece that a retrieval
// client is interested in, as well as specific parameters the client is seeking
// for the retrieval deal
type QueryParams struct {
PayloadCID cid.Cid // optional, query if miner has this cid in this piece. some miners may not be able to respond.
Selector ipld.Node // optional, query if miner has this cid in this piece. some miners may not be able to respond.
MaxPricePerByte BigInt // optional, tell miner uninterested if more expensive than this
MinPaymentInterval uint64 // optional, tell miner uninterested unless payment interval is greater than this
MinPaymentIntervalIncrease uint64 // optional, tell miner uninterested unless payment interval increase is greater than this
}
// Query is a query to a given provider to determine information about a piece
// they may have available for retrieval
type Query struct {
PieceCID []byte // V0
// QueryParams // V1
}
// QueryResponse is a miners response to a given retrieval query
type QueryResponse struct {
Status QueryResponseStatus
//PayloadCIDFound QueryItemStatus // V1 - if a PayloadCid was requested, the result
//SelectorFound QueryItemStatus // V1 - if a Selector was requested, the result
Size uint64 // Total size of piece in bytes
//ExpectedPayloadSize uint64 // V1 - optional, if PayloadCID + selector are specified and miner knows, can offer an expected size
PaymentAddress Address // address to send funds to -- may be different than miner addr
MinPricePerByte BigInt
MaxPaymentInterval uint64
MaxPaymentIntervalIncrease uint64
}
// QueryResponseUndefined is an empty QueryResponse
var QueryResponseUndefined = QueryResponse{}
// PieceRetrievalPrice is the total price to retrieve the piece (size * MinPricePerByte)
func (qr QueryResponse) PieceRetrievalPrice() BigInt {
return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.Size))
}
// PayloadRetrievalPrice is the expected price to retrieve just the given payload
// & selector (V1)
//func (qr QueryResponse) PayloadRetrievalPrice() BigInt {
// return types.BigMul(qr.MinPricePerByte, types.NewInt(qr.ExpectedPayloadSize))
//}
// DealStatus is the status of a retrieval deal returned by a provider
// in a DealResponse
type DealStatus uint64
const (
// DealStatusAccepted means a deal has been accepted by a provider
// and its is ready to proceed with retrieval
DealStatusAccepted DealStatus = iota
// DealStatusFailed indicates something went wrong during a retrieval
DealStatusFailed
// DealStatusRejected indicates the provider rejected a client's deal proposal
// for some reason
DealStatusRejected
// DealStatusUnsealing indicates the provider is currently unsealing the sector
// needed to serve the retrieval deal
DealStatusUnsealing
// DealStatusFundsNeeded indicates the provider is awaiting a payment voucher to
// continue processing the deal
DealStatusFundsNeeded
// DealStatusOngoing indicates the provider is continuing to process a deal
DealStatusOngoing
// DealStatusFundsNeededLastPayment indicates the provider is awaiting funds for
// a final payment in order to complete a deal
DealStatusFundsNeededLastPayment
// DealStatusCompleted indicates a deal is complete
DealStatusCompleted
// DealStatusDealNotFound indicates an update was received for a deal that could
// not be identified
DealStatusDealNotFound
)
// Params are the parameters requested for a retrieval deal proposal
type Params struct {
//PayloadCID cid.Cid // V1
//Selector ipld.Node // V1
PricePerByte BigInt
PaymentInterval uint64
PaymentIntervalIncrease uint64
}
// DealID is an identifier for a retrieval deal (unique to a client)
type DealID uint64
// DealProposal is a proposal for a new retrieval deal
type DealProposal struct {
PieceCID []byte
ID DealID
Params
}
// Block is an IPLD block in bitswap format
type Block struct {
Prefix []byte
Data []byte
}
// DealResponse is a response to a retrieval deal proposal
type DealResponse struct {
Status DealStatus
ID DealID
// payment required to proceed
PaymentOwed BigInt
Message string
Blocks []Block // V0 only
}
// DealPayment is a payment for an in progress retrieval deal
type DealPayment struct {
ID DealID
PaymentChannel address.Address
PaymentVoucher *types.SignedVoucher
}

23
retrievaladapter/provider.go
View File

@ -1,23 +0,0 @@
package retrievaladapter
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
retrievalmarket "github.com/filecoin-project/lotus/retrieval"
)
type retrievalProviderNode struct {
full api.FullNode
}
// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the
// Lotus Node
func NewRetrievalProviderNode(full api.FullNode) retrievalmarket.RetrievalProviderNode {
return &retrievalProviderNode{full}
}
func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievalmarket.SignedVoucher, proof []byte, expectedAmount retrievalmarket.BigInt) (retrievalmarket.BigInt, error) {
return rpn.full.PaychVoucherAdd(ctx, paymentChannel, voucher, proof, expectedAmount)
}

241
storagemarket/bigint.go
View File

@ -1,241 +0,0 @@
// Copied from lotus until this can be extracted into shared types
package storagemarket
import (
"encoding/json"
"fmt"
"io"
"math/big"
"github.com/filecoin-project/lotus/build"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/polydawn/refmt/obj/atlas"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
const BigIntMaxSerializedLen = 128 // is this big enough? or too big?
var TotalFilecoinInt = FromFil(build.TotalFilecoin)
func init() {
cbor.RegisterCborType(atlas.BuildEntry(BigInt{}).Transform().
TransformMarshal(atlas.MakeMarshalTransformFunc(
func(i BigInt) ([]byte, error) {
return i.cborBytes(), nil
})).
TransformUnmarshal(atlas.MakeUnmarshalTransformFunc(
func(x []byte) (BigInt, error) {
return fromCborBytes(x)
})).
Complete())
}
var EmptyInt = BigInt{}
type BigInt struct {
*big.Int
}
func NewInt(i uint64) BigInt {
return BigInt{big.NewInt(0).SetUint64(i)}
}
func FromFil(i uint64) BigInt {
return BigMul(NewInt(i), NewInt(build.FilecoinPrecision))
}
func BigFromBytes(b []byte) BigInt {
i := big.NewInt(0).SetBytes(b)
return BigInt{i}
}
func BigFromString(s string) (BigInt, error) {
v, ok := big.NewInt(0).SetString(s, 10)
if !ok {
return BigInt{}, fmt.Errorf("failed to parse string as a big int")
}
return BigInt{v}, nil
}
func BigMul(a, b BigInt) BigInt {
return BigInt{big.NewInt(0).Mul(a.Int, b.Int)}
}
func BigDiv(a, b BigInt) BigInt {
return BigInt{big.NewInt(0).Div(a.Int, b.Int)}
}
func BigMod(a, b BigInt) BigInt {
return BigInt{big.NewInt(0).Mod(a.Int, b.Int)}
}
func BigAdd(a, b BigInt) BigInt {
return BigInt{big.NewInt(0).Add(a.Int, b.Int)}
}
func BigSub(a, b BigInt) BigInt {
return BigInt{big.NewInt(0).Sub(a.Int, b.Int)}
}
func BigCmp(a, b BigInt) int {
return a.Int.Cmp(b.Int)
}
func (bi BigInt) Nil() bool {
return bi.Int == nil
}
// LessThan returns true if bi < o
func (bi BigInt) LessThan(o BigInt) bool {
return BigCmp(bi, o) < 0
}
// GreaterThan returns true if bi > o
func (bi BigInt) GreaterThan(o BigInt) bool {
return BigCmp(bi, o) > 0
}
// Equals returns true if bi == o
func (bi BigInt) Equals(o BigInt) bool {
return BigCmp(bi, o) == 0
}
func (bi *BigInt) MarshalJSON() ([]byte, error) {
return json.Marshal(bi.String())
}
func (bi *BigInt) UnmarshalJSON(b []byte) error {
var s string
if err := json.Unmarshal(b, &s); err != nil {
return err
}
i, ok := big.NewInt(0).SetString(s, 10)
if !ok {
if string(s) == "<nil>" {
return nil
}
return xerrors.Errorf("failed to parse bigint string: '%s'", string(b))
}
bi.Int = i
return nil
}
func (bi *BigInt) Scan(value interface{}) error {
switch value := value.(type) {
case string:
i, ok := big.NewInt(0).SetString(value, 10)
if !ok {
if value == "<nil>" {
return nil
}
return xerrors.Errorf("failed to parse bigint string: '%s'", value)
}
bi.Int = i
return nil
case int64:
bi.Int = big.NewInt(value)
return nil
default:
return xerrors.Errorf("non-string types unsupported: %T", value)
}
}
func (bi *BigInt) cborBytes() []byte {
if bi.Int == nil {
return []byte{}
}
switch {
case bi.Sign() > 0:
return append([]byte{0}, bi.Bytes()...)
case bi.Sign() < 0:
return append([]byte{1}, bi.Bytes()...)
default: // bi.Sign() == 0:
return []byte{}
}
}
func fromCborBytes(buf []byte) (BigInt, error) {
if len(buf) == 0 {
return NewInt(0), nil
}
var negative bool
switch buf[0] {
case 0:
negative = false
case 1:
negative = true
default:
return EmptyInt, fmt.Errorf("big int prefix should be either 0 or 1, got %d", buf[0])
}
i := big.NewInt(0).SetBytes(buf[1:])
if negative {
i.Neg(i)
}
return BigInt{i}, nil
}
func (bi *BigInt) MarshalCBOR(w io.Writer) error {
if bi.Int == nil {
zero := NewInt(0)
return zero.MarshalCBOR(w)
}
enc := bi.cborBytes()
header := cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(enc)))
if _, err := w.Write(header); err != nil {
return err
}
if _, err := w.Write(enc); err != nil {
return err
}
return nil
}
func (bi *BigInt) UnmarshalCBOR(br io.Reader) error {
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajByteString {
return fmt.Errorf("cbor input for fil big int was not a byte string (%x)", maj)
}
if extra == 0 {
bi.Int = big.NewInt(0)
return nil
}
if extra > BigIntMaxSerializedLen {
return fmt.Errorf("big integer byte array too long")
}
buf := make([]byte, extra)
if _, err := io.ReadFull(br, buf); err != nil {
return err
}
i, err := fromCborBytes(buf)
if err != nil {
return err
}
*bi = i
return nil
}

352
storagemarket/cbor_gen.go
View File

@ -1,352 +0,0 @@
package storagemarket
import (
"fmt"
"io"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
var _ = xerrors.Errorf
func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{136}); err != nil {
return err
}
// t.ProposalCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
}
// t.Proposal (actors.StorageDealProposal) (struct)
if err := t.Proposal.MarshalCBOR(w); err != nil {
return err
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Miner (peer.ID) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Miner)); err != nil {
return err
}
// t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.PayloadCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.PayloadCid); err != nil {
return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err)
}
// t.PublishMessage (cid.Cid) (struct)
if t.PublishMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.PublishMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err)
}
}
return nil
}
func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 8 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.ProposalCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
}
t.ProposalCid = c
}
// t.Proposal (actors.StorageDealProposal) (struct)
{
if err := t.Proposal.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Miner (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Miner = peer.ID(sval)
}
// t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.PayloadCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err)
}
t.PayloadCid = c
}
// t.PublishMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err)
}
t.PublishMessage = &c
}
}
return nil
}
func (t *MinerDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{136}); err != nil {
return err
}
// t.ProposalCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
}
// t.Proposal (actors.StorageDealProposal) (struct)
if err := t.Proposal.MarshalCBOR(w); err != nil {
return err
}
// t.Miner (peer.ID) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Miner)); err != nil {
return err
}
// t.Client (peer.ID) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Client)); err != nil {
return err
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Ref (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Ref); err != nil {
return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
return nil
}
func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 8 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.ProposalCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
}
t.ProposalCid = c
}
// t.Proposal (actors.StorageDealProposal) (struct)
{
if err := t.Proposal.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Miner (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Miner = peer.ID(sval)
}
// t.Client (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Client = peer.ID(sval)
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Ref (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
}
t.Ref = c
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
return nil
}

199
storagemarket/types.go
View File

@ -1,199 +0,0 @@
package storagemarket
import (
"context"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
)
const DealProtocolID = "/fil/storage/mk/1.0.1"
const AskProtocolID = "/fil/storage/ask/1.0.1"
// type shims - used during migration into separate module
type Balance = actors.StorageParticipantBalance
type DealID uint64
type Signature = types.Signature
type StorageDeal = actors.OnChainDeal
type StorageAsk = types.SignedStorageAsk
type StateKey = *types.TipSet
type Epoch uint64
type TokenAmount BigInt
// Duplicated from deals package for now
type MinerDeal struct {
ProposalCid cid.Cid
Proposal actors.StorageDealProposal
Miner peer.ID
Client peer.ID
State api.DealState
Ref cid.Cid
DealID uint64
SectorID uint64 // Set when sm >= DealStaged
}
type ClientDeal struct {
ProposalCid cid.Cid
Proposal actors.StorageDealProposal
State api.DealState
Miner peer.ID
MinerWorker address.Address
DealID uint64
PayloadCid cid.Cid
PublishMessage *cid.Cid
}
// The interface provided for storage providers
type StorageProvider interface {
Run(ctx context.Context, host host.Host)
Stop()
AddAsk(price TokenAmount, ttlsecs int64) error
// ListAsks lists current asks
ListAsks(addr address.Address) []*StorageAsk
// ListDeals lists on-chain deals associated with this provider
ListDeals(ctx context.Context) ([]StorageDeal, error)
// ListIncompleteDeals lists deals that are in progress or rejected
ListIncompleteDeals() ([]MinerDeal, error)
// AddStorageCollateral adds storage collateral
AddStorageCollateral(ctx context.Context, amount TokenAmount) error
// GetStorageCollateral returns the current collateral balance
GetStorageCollateral(ctx context.Context) (Balance, error)
}
// Node dependencies for a StorageProvider
type StorageProviderNode interface {
MostRecentStateId(ctx context.Context) (StateKey, error)
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error
// Ensures that a storage market participant has a certain amount of available funds
EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error
// GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients.
GetBalance(ctx context.Context, addr address.Address) (Balance, error)
// Publishes deal on chain
PublishDeals(ctx context.Context, deal MinerDeal) (DealID, cid.Cid, error)
// ListProviderDeals lists all deals associated with a storage provider
ListProviderDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error)
// Called when a deal is complete and on chain, and data has been transferred and is ready to be added to a sector
// returns sector id
OnDealComplete(ctx context.Context, deal MinerDeal, piecePath string) (uint64, error)
// returns the worker address associated with a miner
GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error)
// Signs bytes
SignBytes(ctx context.Context, signer address.Address, b []byte) (*types.Signature, error)
}
type DealSectorCommittedCallback func(error)
// Node dependencies for a StorageClient
type StorageClientNode interface {
MostRecentStateId(ctx context.Context) (StateKey, error)
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
AddFunds(ctx context.Context, addr address.Address, amount TokenAmount) error
EnsureFunds(ctx context.Context, addr address.Address, amount TokenAmount) error
// GetBalance returns locked/unlocked for a storage participant. Used by both providers and clients.
GetBalance(ctx context.Context, addr address.Address) (Balance, error)
//// ListClientDeals lists all on-chain deals associated with a storage client
ListClientDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error)
// GetProviderInfo returns information about a single storage provider
//GetProviderInfo(stateId StateID, addr Address) *StorageProviderInfo
// GetStorageProviders returns information about known miners
ListStorageProviders(ctx context.Context) ([]*StorageProviderInfo, error)
// Subscribes to storage market actor state changes for a given address.
// TODO: Should there be a timeout option for this? In the case that we are waiting for funds to be deposited and it never happens?
//SubscribeStorageMarketEvents(addr Address, handler StorageMarketEventHandler) (SubID, error)
// Cancels a subscription
//UnsubscribeStorageMarketEvents(subId SubID)
ValidatePublishedDeal(ctx context.Context, deal ClientDeal) (uint64, error)
// SignProposal signs a proposal
SignProposal(ctx context.Context, signer address.Address, proposal *actors.StorageDealProposal) error
GetDefaultWalletAddress(ctx context.Context) (address.Address, error)
OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb DealSectorCommittedCallback) error
ValidateAskSignature(ask *StorageAsk) error
}
type StorageClientProofs interface {
//GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (CommP, error)
}
// Closely follows the MinerInfo struct in the spec
type StorageProviderInfo struct {
Address address.Address // actor address
Owner address.Address
Worker address.Address // signs messages
SectorSize uint64
PeerID peer.ID
// probably more like how much storage power, available collateral etc
}
type ProposeStorageDealResult struct {
ProposalCid cid.Cid
}
// The interface provided by the module to the outside world for storage clients.
type StorageClient interface {
Run(ctx context.Context)
Stop()
// ListProviders queries chain state and returns active storage providers
ListProviders(ctx context.Context) (<-chan StorageProviderInfo, error)
// ListDeals lists on-chain deals associated with this provider
ListDeals(ctx context.Context, addr address.Address) ([]StorageDeal, error)
// ListInProgressDeals lists deals that are in progress or rejected
ListInProgressDeals(ctx context.Context) ([]ClientDeal, error)
// ListInProgressDeals lists deals that are in progress or rejected
GetInProgressDeal(ctx context.Context, cid cid.Cid) (ClientDeal, error)
// GetAsk returns the current ask for a storage provider
GetAsk(ctx context.Context, info StorageProviderInfo) (*StorageAsk, error)
//// FindStorageOffers lists providers and queries them to find offers that satisfy some criteria based on price, duration, etc.
//FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer
// ProposeStorageDeal initiates deal negotiation with a Storage Provider
ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, payloadCid cid.Cid, proposalExpiration Epoch, duration Epoch, price TokenAmount, collateral TokenAmount) (*ProposeStorageDealResult, error)
// GetPaymentEscrow returns the current funds available for deal payment
GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error)
// AddStorageCollateral adds storage collateral
AddPaymentEscrow(ctx context.Context, addr address.Address, amount TokenAmount) error
}