cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

v1.27.0-a #10

Closed
jonathanface wants to merge 473 commits from v1.27.0-a into master
pull from: v1.27.0-a
merge into: cerc-io:master
Conversation 0 Commits 473 Files Changed 452 +277 -15
10 changed files with 277 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0337d0198f - Show all commits

10
api/api_lp.go
View File

@ -1,10 +1,18 @@
package api
import "context"
import (
"context"
"net/http"
"net/url"
"github.com/filecoin-project/go-address"
)
type LotusProvider interface {
Version(context.Context) (Version, error) //perm:admin
AllocatePieceToSector(ctx context.Context, maddr address.Address, piece PieceDealInfo, rawSize int64, source url.URL, header http.Header) (SectorOffset, error) //perm:write
// Trigger shutdown
Shutdown(context.Context) error //perm:admin
}

15
api/proxy_gen.go
View File

@ -5,6 +5,8 @@ package api
import (
"context"
"encoding/json"
"net/http"
"net/url"
"time"
"github.com/google/uuid"
@ -832,6 +834,8 @@ type LotusProviderStruct struct {
}
type LotusProviderMethods struct {
AllocatePieceToSector func(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) `perm:"write"`
Shutdown func(p0 context.Context) error `perm:"admin"`
Version func(p0 context.Context) (Version, error) `perm:"admin"`
@ -5201,6 +5205,17 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) {
return "", ErrNotSupported
}
func (s *LotusProviderStruct) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
if s.Internal.AllocatePieceToSector == nil {
return *new(SectorOffset), ErrNotSupported
}
return s.Internal.AllocatePieceToSector(p0, p1, p2, p3, p4, p5)
}
func (s *LotusProviderStub) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
return *new(SectorOffset), ErrNotSupported
}
func (s *LotusProviderStruct) Shutdown(p0 context.Context) error {
if s.Internal.Shutdown == nil {
return ErrNotSupported

16
cmd/lotus-provider/rpc/rpc.go
View File

@ -7,6 +7,7 @@ import (
"encoding/json"
"net"
"net/http"
"net/url"
"time"
"github.com/gbrlsnchs/jwt/v3"
@ -16,6 +17,7 @@ import (
"golang.org/x/sync/errgroup"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/go-jsonrpc/auth"
@ -24,6 +26,7 @@ import (
"github.com/filecoin-project/lotus/lib/rpcenc"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/metrics/proxy"
"github.com/filecoin-project/lotus/provider/lpmarket"
"github.com/filecoin-project/lotus/provider/lpweb"
"github.com/filecoin-project/lotus/storage/paths"
)
@ -72,6 +75,12 @@ func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
return api.ProviderAPIVersion0, nil
}
func (p *ProviderAPI) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
di := lpmarket.NewPieceIngester(p.Deps.DB, p.Deps.Full)
return di.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
}
// Trigger shutdown
func (p *ProviderAPI) Shutdown(context.Context) error {
close(p.ShutdownChan)
@ -89,13 +98,6 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
fh.ServeHTTP(w, r)
}
// local APIs
{
// debugging
mux := mux.NewRouter()
mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
}
var authVerify func(context.Context, string) ([]auth.Permission, error)
{

70
documentation/en/api-v0-methods-provider.md
View File

@ -2,6 +2,8 @@
* [](#)
* [Shutdown](#Shutdown)
* [Version](#Version)
* [Allocate](#Allocate)
* [AllocatePieceToSector](#AllocatePieceToSector)
##
@ -23,3 +25,71 @@ Inputs: `null`
Response: `131840`
## Allocate
### AllocatePieceToSector
Perms: write
Inputs:
```json
[
"f01234",
{
"PublishCid": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
"DealID": 5432,
"DealProposal": {
"PieceCID": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
"PieceSize": 1032,
"VerifiedDeal": true,
"Client": "f01234",
"Provider": "f01234",
"Label": "",
"StartEpoch": 10101,
"EndEpoch": 10101,
"StoragePricePerEpoch": "0",
"ProviderCollateral": "0",
"ClientCollateral": "0"
},
"DealSchedule": {
"StartEpoch": 10101,
"EndEpoch": 10101
},
"KeepUnsealed": true
},
9,
{
"Scheme": "string value",
"Opaque": "string value",
"User": {},
"Host": "string value",
"Path": "string value",
"RawPath": "string value",
"OmitHost": true,
"ForceQuery": true,
"RawQuery": "string value",
"Fragment": "string value",
"RawFragment": "string value"
},
{
"Authorization": [
"Bearer ey.."
]
}
]
```
Response:
```json
{
"Sector": 9,
"Offset": 1032
}
```

13
lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql
View File

@ -79,6 +79,19 @@ create table sectors_sdr_initial_pieces (
piece_cid text not null,
piece_size bigint not null,
-- data source
data_url text not null,
data_headers jsonb not null default '{}',
data_raw_size bigint not null,
data_delete_on_finalize bool not null,
-- deal info
f05_publish_cid text,
f05_deal_id bigint,
f05_deal_proposal jsonb,
f05_deal_start_epoch bigint,
f05_deal_end_epoch bigint,
-- foreign key
foreign key (sp_id, sector_number) references sectors_sdr_pipeline (sp_id, sector_number) on delete cascade,

132
provider/lpmarket/deal_ingest.go Normal file
View File

@ -0,0 +1,132 @@
package lpmarket
import (
"context"
"encoding/json"
"net/http"
"net/url"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/provider/lpseal"
)
type Ingester interface {
AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error)
}
type PieceIngesterApi interface {
StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)
StateMinerAllocated(ctx context.Context, a address.Address, key types.TipSetKey) (*bitfield.BitField, error)
StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error)
}
type PieceIngester struct {
db *harmonydb.DB
api PieceIngesterApi
}
func NewPieceIngester(db *harmonydb.DB, api PieceIngesterApi) *PieceIngester {
return &PieceIngester{db: db, api: api}
}
func (p *PieceIngester) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
mi, err := p.api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return api.SectorOffset{}, err
}
if piece.DealProposal.PieceSize != abi.PaddedPieceSize(mi.SectorSize) {
return api.SectorOffset{}, xerrors.Errorf("only full sector pieces supported for now")
}
// check raw size
if piece.DealProposal.PieceSize != padreader.PaddedSize(uint64(rawSize)).Padded() {
return api.SectorOffset{}, xerrors.Errorf("raw size doesn't match padded piece size")
}
// add initial piece + to a sector
nv, err := p.api.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return api.SectorOffset{}, xerrors.Errorf("getting network version: %w", err)
}
synth := false // todo synthetic porep config
spt, err := miner.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType, synth)
if err != nil {
return api.SectorOffset{}, xerrors.Errorf("getting seal proof type: %w", err)
}
num, err := lpseal.AllocateSectorNumbers(ctx, p.api, p.db, maddr, 1, func(tx *harmonydb.Tx, numbers []abi.SectorNumber) (bool, error) {
if len(numbers) != 1 {
return false, xerrors.Errorf("expected one sector number")
}
n := numbers[0]
_, err := tx.Exec("insert into sectors_sdr_pipeline (sp_id, sector_number, reg_seal_proof) values ($1, $2, $3)", maddr, n, spt)
if err != nil {
return false, xerrors.Errorf("inserting into sectors_sdr_pipeline: %w", err)
}
dataHdrJson, err := json.Marshal(header)
if err != nil {
return false, xerrors.Errorf("json.Marshal(header): %w", err)
}
dealProposalJson, err := json.Marshal(piece.DealProposal)
if err != nil {
return false, xerrors.Errorf("json.Marshal(piece.DealProposal): %w", err)
}
_, err = tx.Exec(`INSERT INTO sectors_sdr_initial_pieces (sp_id,
sector_number,
piece_index,
piece_cid,
piece_size,
data_url,
data_headers,
data_raw_size,
data_delete_on_finalize,
f05_publish_cid,
f05_deal_id,
f05_deal_proposal,
f05_deal_start_epoch,
f05_deal_end_epoch) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
maddr, n, 0,
piece.DealProposal.PieceCID, piece.DealProposal.PieceSize,
source.String(), dataHdrJson, rawSize, true,
piece.PublishCid, piece.DealID, dealProposalJson, piece.DealSchedule.StartEpoch, piece.DealSchedule.EndEpoch)
if err != nil {
return false, xerrors.Errorf("inserting into sectors_sdr_initial_pieces: %w", err)
}
return true, nil
})
if err != nil {
return api.SectorOffset{}, xerrors.Errorf("allocating sector numbers: %w", err)
}
if len(num) != 1 {
return api.SectorOffset{}, xerrors.Errorf("expected one sector number")
}
// After we insert the piece/sector_pipeline entries, the lpseal/poller will take it from here
return api.SectorOffset{
Sector: num[0],
Offset: 0,
}, nil
}

7
provider/lpmarket/fakelm/iface.go
View File

@ -2,12 +2,15 @@ package fakelm
import (
"context"
"io"
"github.com/google/uuid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/google/uuid"
"io"
)
// MinimalLMApi is a subset of the LotusMiner API that is exposed by lotus-provider

19
provider/lpmarket/fakelm/lmimpl.go
View File

@ -2,15 +2,21 @@ package fakelm
import (
"context"
"io"
"net/http"
"net/url"
"github.com/google/uuid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/provider/lpmarket"
"github.com/filecoin-project/lotus/storage/paths"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
"github.com/google/uuid"
"golang.org/x/xerrors"
"io"
)
type LMRPCProvider struct {
@ -20,6 +26,8 @@ type LMRPCProvider struct {
minerID abi.ActorID
ssize abi.SectorSize
pi lpmarket.Ingester
}
func (l *LMRPCProvider) ActorAddress(ctx context.Context) (address.Address, error) {
@ -259,6 +267,11 @@ func (l *LMRPCProvider) SectorAddPieceToAny(ctx context.Context, size abi.Unpadd
return api.SectorOffset{}, xerrors.Errorf("only full-sector pieces are supported")
}
return api.SectorOffset{}, xerrors.Errorf("not supported, use AllocatePieceToSector")
}
func (l *LMRPCProvider) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
return l.pi.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
}
var _ MinimalLMApi = &LMRPCProvider{}

5
provider/lpseal/poller_commit_msg.go
View File

@ -2,13 +2,16 @@ package lpseal
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"golang.org/x/xerrors"
)
func (s *SealPoller) pollStartCommitMsg(ctx context.Context, task pollTask) {

5
provider/lpseal/poller_precommit_msg.go
View File

@ -2,14 +2,17 @@ package lpseal
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"golang.org/x/xerrors"
)
func (s *SealPoller) pollStartPrecommitMsg(ctx context.Context, task pollTask) {