lpseal: Basic deal-accepting logic
This commit is contained in:
parent
29584e0f31
commit
0337d0198f
@ -1,10 +1,18 @@
|
||||
package api
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
)
|
||||
|
||||
type LotusProvider interface {
|
||||
Version(context.Context) (Version, error) //perm:admin
|
||||
|
||||
AllocatePieceToSector(ctx context.Context, maddr address.Address, piece PieceDealInfo, rawSize int64, source url.URL, header http.Header) (SectorOffset, error) //perm:write
|
||||
|
||||
// Trigger shutdown
|
||||
Shutdown(context.Context) error //perm:admin
|
||||
}
|
||||
|
@ -5,6 +5,8 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@ -832,6 +834,8 @@ type LotusProviderStruct struct {
|
||||
}
|
||||
|
||||
type LotusProviderMethods struct {
|
||||
AllocatePieceToSector func(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) `perm:"write"`
|
||||
|
||||
Shutdown func(p0 context.Context) error `perm:"admin"`
|
||||
|
||||
Version func(p0 context.Context) (Version, error) `perm:"admin"`
|
||||
@ -5201,6 +5205,17 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) {
|
||||
return "", ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
|
||||
if s.Internal.AllocatePieceToSector == nil {
|
||||
return *new(SectorOffset), ErrNotSupported
|
||||
}
|
||||
return s.Internal.AllocatePieceToSector(p0, p1, p2, p3, p4, p5)
|
||||
}
|
||||
|
||||
func (s *LotusProviderStub) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
|
||||
return *new(SectorOffset), ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *LotusProviderStruct) Shutdown(p0 context.Context) error {
|
||||
if s.Internal.Shutdown == nil {
|
||||
return ErrNotSupported
|
||||
|
@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/gbrlsnchs/jwt/v3"
|
||||
@ -16,6 +17,7 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-jsonrpc"
|
||||
"github.com/filecoin-project/go-jsonrpc/auth"
|
||||
|
||||
@ -24,6 +26,7 @@ import (
|
||||
"github.com/filecoin-project/lotus/lib/rpcenc"
|
||||
"github.com/filecoin-project/lotus/metrics"
|
||||
"github.com/filecoin-project/lotus/metrics/proxy"
|
||||
"github.com/filecoin-project/lotus/provider/lpmarket"
|
||||
"github.com/filecoin-project/lotus/provider/lpweb"
|
||||
"github.com/filecoin-project/lotus/storage/paths"
|
||||
)
|
||||
@ -72,6 +75,12 @@ func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
|
||||
return api.ProviderAPIVersion0, nil
|
||||
}
|
||||
|
||||
func (p *ProviderAPI) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
|
||||
di := lpmarket.NewPieceIngester(p.Deps.DB, p.Deps.Full)
|
||||
|
||||
return di.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
|
||||
}
|
||||
|
||||
// Trigger shutdown
|
||||
func (p *ProviderAPI) Shutdown(context.Context) error {
|
||||
close(p.ShutdownChan)
|
||||
@ -89,13 +98,6 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
|
||||
|
||||
fh.ServeHTTP(w, r)
|
||||
}
|
||||
// local APIs
|
||||
{
|
||||
// debugging
|
||||
mux := mux.NewRouter()
|
||||
mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
|
||||
mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
|
||||
}
|
||||
|
||||
var authVerify func(context.Context, string) ([]auth.Permission, error)
|
||||
{
|
||||
|
@ -2,6 +2,8 @@
|
||||
* [](#)
|
||||
* [Shutdown](#Shutdown)
|
||||
* [Version](#Version)
|
||||
* [Allocate](#Allocate)
|
||||
* [AllocatePieceToSector](#AllocatePieceToSector)
|
||||
##
|
||||
|
||||
|
||||
@ -23,3 +25,71 @@ Inputs: `null`
|
||||
|
||||
Response: `131840`
|
||||
|
||||
## Allocate
|
||||
|
||||
|
||||
### AllocatePieceToSector
|
||||
|
||||
|
||||
Perms: write
|
||||
|
||||
Inputs:
|
||||
```json
|
||||
[
|
||||
"f01234",
|
||||
{
|
||||
"PublishCid": {
|
||||
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
|
||||
},
|
||||
"DealID": 5432,
|
||||
"DealProposal": {
|
||||
"PieceCID": {
|
||||
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
|
||||
},
|
||||
"PieceSize": 1032,
|
||||
"VerifiedDeal": true,
|
||||
"Client": "f01234",
|
||||
"Provider": "f01234",
|
||||
"Label": "",
|
||||
"StartEpoch": 10101,
|
||||
"EndEpoch": 10101,
|
||||
"StoragePricePerEpoch": "0",
|
||||
"ProviderCollateral": "0",
|
||||
"ClientCollateral": "0"
|
||||
},
|
||||
"DealSchedule": {
|
||||
"StartEpoch": 10101,
|
||||
"EndEpoch": 10101
|
||||
},
|
||||
"KeepUnsealed": true
|
||||
},
|
||||
9,
|
||||
{
|
||||
"Scheme": "string value",
|
||||
"Opaque": "string value",
|
||||
"User": {},
|
||||
"Host": "string value",
|
||||
"Path": "string value",
|
||||
"RawPath": "string value",
|
||||
"OmitHost": true,
|
||||
"ForceQuery": true,
|
||||
"RawQuery": "string value",
|
||||
"Fragment": "string value",
|
||||
"RawFragment": "string value"
|
||||
},
|
||||
{
|
||||
"Authorization": [
|
||||
"Bearer ey.."
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Response:
|
||||
```json
|
||||
{
|
||||
"Sector": 9,
|
||||
"Offset": 1032
|
||||
}
|
||||
```
|
||||
|
||||
|
@ -79,6 +79,19 @@ create table sectors_sdr_initial_pieces (
|
||||
piece_cid text not null,
|
||||
piece_size bigint not null,
|
||||
|
||||
-- data source
|
||||
data_url text not null,
|
||||
data_headers jsonb not null default '{}',
|
||||
data_raw_size bigint not null,
|
||||
data_delete_on_finalize bool not null,
|
||||
|
||||
-- deal info
|
||||
f05_publish_cid text,
|
||||
f05_deal_id bigint,
|
||||
f05_deal_proposal jsonb,
|
||||
f05_deal_start_epoch bigint,
|
||||
f05_deal_end_epoch bigint,
|
||||
|
||||
-- foreign key
|
||||
foreign key (sp_id, sector_number) references sectors_sdr_pipeline (sp_id, sector_number) on delete cascade,
|
||||
|
||||
|
132
provider/lpmarket/deal_ingest.go
Normal file
132
provider/lpmarket/deal_ingest.go
Normal file
@ -0,0 +1,132 @@
|
||||
package lpmarket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-bitfield"
|
||||
"github.com/filecoin-project/go-padreader"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/network"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
"github.com/filecoin-project/lotus/provider/lpseal"
|
||||
)
|
||||
|
||||
type Ingester interface {
|
||||
AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error)
|
||||
}
|
||||
|
||||
type PieceIngesterApi interface {
|
||||
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)
|
||||
StateMinerAllocated(ctx context.Context, a address.Address, key types.TipSetKey) (*bitfield.BitField, error)
|
||||
StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error)
|
||||
}
|
||||
|
||||
type PieceIngester struct {
|
||||
db *harmonydb.DB
|
||||
api PieceIngesterApi
|
||||
}
|
||||
|
||||
func NewPieceIngester(db *harmonydb.DB, api PieceIngesterApi) *PieceIngester {
|
||||
return &PieceIngester{db: db, api: api}
|
||||
}
|
||||
|
||||
func (p *PieceIngester) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
|
||||
mi, err := p.api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return api.SectorOffset{}, err
|
||||
}
|
||||
|
||||
if piece.DealProposal.PieceSize != abi.PaddedPieceSize(mi.SectorSize) {
|
||||
return api.SectorOffset{}, xerrors.Errorf("only full sector pieces supported for now")
|
||||
}
|
||||
|
||||
// check raw size
|
||||
if piece.DealProposal.PieceSize != padreader.PaddedSize(uint64(rawSize)).Padded() {
|
||||
return api.SectorOffset{}, xerrors.Errorf("raw size doesn't match padded piece size")
|
||||
}
|
||||
|
||||
// add initial piece + to a sector
|
||||
nv, err := p.api.StateNetworkVersion(ctx, types.EmptyTSK)
|
||||
if err != nil {
|
||||
return api.SectorOffset{}, xerrors.Errorf("getting network version: %w", err)
|
||||
}
|
||||
|
||||
synth := false // todo synthetic porep config
|
||||
spt, err := miner.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType, synth)
|
||||
if err != nil {
|
||||
return api.SectorOffset{}, xerrors.Errorf("getting seal proof type: %w", err)
|
||||
}
|
||||
|
||||
num, err := lpseal.AllocateSectorNumbers(ctx, p.api, p.db, maddr, 1, func(tx *harmonydb.Tx, numbers []abi.SectorNumber) (bool, error) {
|
||||
if len(numbers) != 1 {
|
||||
return false, xerrors.Errorf("expected one sector number")
|
||||
}
|
||||
n := numbers[0]
|
||||
|
||||
_, err := tx.Exec("insert into sectors_sdr_pipeline (sp_id, sector_number, reg_seal_proof) values ($1, $2, $3)", maddr, n, spt)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("inserting into sectors_sdr_pipeline: %w", err)
|
||||
}
|
||||
|
||||
dataHdrJson, err := json.Marshal(header)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("json.Marshal(header): %w", err)
|
||||
}
|
||||
|
||||
dealProposalJson, err := json.Marshal(piece.DealProposal)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("json.Marshal(piece.DealProposal): %w", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(`INSERT INTO sectors_sdr_initial_pieces (sp_id,
|
||||
sector_number,
|
||||
piece_index,
|
||||
|
||||
piece_cid,
|
||||
piece_size,
|
||||
|
||||
data_url,
|
||||
data_headers,
|
||||
data_raw_size,
|
||||
data_delete_on_finalize,
|
||||
|
||||
f05_publish_cid,
|
||||
f05_deal_id,
|
||||
f05_deal_proposal,
|
||||
f05_deal_start_epoch,
|
||||
f05_deal_end_epoch) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
|
||||
maddr, n, 0,
|
||||
piece.DealProposal.PieceCID, piece.DealProposal.PieceSize,
|
||||
source.String(), dataHdrJson, rawSize, true,
|
||||
piece.PublishCid, piece.DealID, dealProposalJson, piece.DealSchedule.StartEpoch, piece.DealSchedule.EndEpoch)
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("inserting into sectors_sdr_initial_pieces: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return api.SectorOffset{}, xerrors.Errorf("allocating sector numbers: %w", err)
|
||||
}
|
||||
|
||||
if len(num) != 1 {
|
||||
return api.SectorOffset{}, xerrors.Errorf("expected one sector number")
|
||||
}
|
||||
|
||||
// After we insert the piece/sector_pipeline entries, the lpseal/poller will take it from here
|
||||
|
||||
return api.SectorOffset{
|
||||
Sector: num[0],
|
||||
Offset: 0,
|
||||
}, nil
|
||||
}
|
@ -2,12 +2,15 @@ package fakelm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
)
|
||||
|
||||
// MinimalLMApi is a subset of the LotusMiner API that is exposed by lotus-provider
|
||||
|
@ -2,15 +2,21 @@ package fakelm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
"github.com/filecoin-project/lotus/provider/lpmarket"
|
||||
"github.com/filecoin-project/lotus/storage/paths"
|
||||
sealing "github.com/filecoin-project/lotus/storage/pipeline"
|
||||
"github.com/filecoin-project/lotus/storage/sealer/storiface"
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/xerrors"
|
||||
"io"
|
||||
)
|
||||
|
||||
type LMRPCProvider struct {
|
||||
@ -20,6 +26,8 @@ type LMRPCProvider struct {
|
||||
minerID abi.ActorID
|
||||
|
||||
ssize abi.SectorSize
|
||||
|
||||
pi lpmarket.Ingester
|
||||
}
|
||||
|
||||
func (l *LMRPCProvider) ActorAddress(ctx context.Context) (address.Address, error) {
|
||||
@ -259,6 +267,11 @@ func (l *LMRPCProvider) SectorAddPieceToAny(ctx context.Context, size abi.Unpadd
|
||||
return api.SectorOffset{}, xerrors.Errorf("only full-sector pieces are supported")
|
||||
}
|
||||
|
||||
return api.SectorOffset{}, xerrors.Errorf("not supported, use AllocatePieceToSector")
|
||||
}
|
||||
|
||||
func (l *LMRPCProvider) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
|
||||
return l.pi.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
|
||||
}
|
||||
|
||||
var _ MinimalLMApi = &LMRPCProvider{}
|
||||
|
@ -2,13 +2,16 @@ package lpseal
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func (s *SealPoller) pollStartCommitMsg(ctx context.Context, task pollTask) {
|
||||
|
@ -2,14 +2,17 @@ package lpseal
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-address"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/exitcode"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/actors/policy"
|
||||
"github.com/filecoin-project/lotus/chain/types"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
|
||||
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
func (s *SealPoller) pollStartPrecommitMsg(ctx context.Context, task pollTask) {
|
||||
|
Loading…
Reference in New Issue
Block a user