api/api_lp.go

@@ -1,10 +1,18 @@
 package api
 
-import "context"
+import (
+	"context"
+	"net/http"
+	"net/url"
+
+	"github.com/filecoin-project/go-address"
+)
 
 type LotusProvider interface {
 	Version(context.Context) (Version, error) //perm:admin
 
+	AllocatePieceToSector(ctx context.Context, maddr address.Address, piece PieceDealInfo, rawSize int64, source url.URL, header http.Header) (SectorOffset, error) //perm:write
+
 	// Trigger shutdown
 	Shutdown(context.Context) error //perm:admin
 }

api/proxy_gen.go

@@ -5,6 +5,8 @@ package api
 import (
 	"context"
 	"encoding/json"
+	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/google/uuid"
@@ -832,6 +834,8 @@ type LotusProviderStruct struct {
 }
 
 type LotusProviderMethods struct {
+	AllocatePieceToSector func(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) `perm:"write"`
+
 	Shutdown func(p0 context.Context) error `perm:"admin"`
 
 	Version func(p0 context.Context) (Version, error) `perm:"admin"`
@@ -5201,6 +5205,17 @@ func (s *GatewayStub) Web3ClientVersion(p0 context.Context) (string, error) {
 	return "", ErrNotSupported
 }
 
+func (s *LotusProviderStruct) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
+	if s.Internal.AllocatePieceToSector == nil {
+		return *new(SectorOffset), ErrNotSupported
+	}
+	return s.Internal.AllocatePieceToSector(p0, p1, p2, p3, p4, p5)
+}
+
+func (s *LotusProviderStub) AllocatePieceToSector(p0 context.Context, p1 address.Address, p2 PieceDealInfo, p3 int64, p4 url.URL, p5 http.Header) (SectorOffset, error) {
+	return *new(SectorOffset), ErrNotSupported
+}
+
 func (s *LotusProviderStruct) Shutdown(p0 context.Context) error {
 	if s.Internal.Shutdown == nil {
 		return ErrNotSupported

cmd/lotus-provider/rpc/rpc.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/gbrlsnchs/jwt/v3"
@@ -16,6 +17,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
@@ -24,6 +26,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/rpcenc"
 	"github.com/filecoin-project/lotus/metrics"
 	"github.com/filecoin-project/lotus/metrics/proxy"
+	"github.com/filecoin-project/lotus/provider/lpmarket"
 	"github.com/filecoin-project/lotus/provider/lpweb"
 	"github.com/filecoin-project/lotus/storage/paths"
 )
@@ -72,6 +75,12 @@ func (p *ProviderAPI) Version(context.Context) (api.Version, error) {
 	return api.ProviderAPIVersion0, nil
 }
 
+func (p *ProviderAPI) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
+	di := lpmarket.NewPieceIngester(p.Deps.DB, p.Deps.Full)
+
+	return di.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
+}
+
 // Trigger shutdown
 func (p *ProviderAPI) Shutdown(context.Context) error {
 	close(p.ShutdownChan)
@@ -89,13 +98,6 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
 
 		fh.ServeHTTP(w, r)
 	}
-	// local APIs
-	{
-		// debugging
-		mux := mux.NewRouter()
-		mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
-		mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
-	}
 
 	var authVerify func(context.Context, string) ([]auth.Permission, error)
 	{

documentation/en/api-v0-methods-provider.md

@@ -2,6 +2,8 @@
 * [](#)
   * [Shutdown](#Shutdown)
   * [Version](#Version)
+* [Allocate](#Allocate)
+  * [AllocatePieceToSector](#AllocatePieceToSector)
 ## 
 
 
@@ -23,3 +25,71 @@ Inputs: `null`
 
 Response: `131840`
 
+## Allocate
+
+
+### AllocatePieceToSector
+
+
+Perms: write
+
+Inputs:
+```json
+[
+  "f01234",
+  {
+    "PublishCid": {
+      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+    },
+    "DealID": 5432,
+    "DealProposal": {
+      "PieceCID": {
+        "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+      },
+      "PieceSize": 1032,
+      "VerifiedDeal": true,
+      "Client": "f01234",
+      "Provider": "f01234",
+      "Label": "",
+      "StartEpoch": 10101,
+      "EndEpoch": 10101,
+      "StoragePricePerEpoch": "0",
+      "ProviderCollateral": "0",
+      "ClientCollateral": "0"
+    },
+    "DealSchedule": {
+      "StartEpoch": 10101,
+      "EndEpoch": 10101
+    },
+    "KeepUnsealed": true
+  },
+  9,
+  {
+    "Scheme": "string value",
+    "Opaque": "string value",
+    "User": {},
+    "Host": "string value",
+    "Path": "string value",
+    "RawPath": "string value",
+    "OmitHost": true,
+    "ForceQuery": true,
+    "RawQuery": "string value",
+    "Fragment": "string value",
+    "RawFragment": "string value"
+  },
+  {
+    "Authorization": [
+      "Bearer ey.."
+    ]
+  }
+]
+```
+
+Response:
+```json
+{
+  "Sector": 9,
+  "Offset": 1032
+}
+```
+

lib/harmony/harmonydb/sql/20231217-sdr-pipeline.sql

@@ -79,6 +79,19 @@ create table sectors_sdr_initial_pieces (
     piece_cid text not null,
     piece_size bigint not null,
 
+    -- data source
+    data_url text not null,
+    data_headers jsonb not null default '{}',
+    data_raw_size bigint not null,
+    data_delete_on_finalize bool not null,
+
+    -- deal info
+    f05_publish_cid text,
+    f05_deal_id bigint,
+    f05_deal_proposal jsonb,
+    f05_deal_start_epoch bigint,
+    f05_deal_end_epoch bigint,
+
     -- foreign key
     foreign key (sp_id, sector_number) references sectors_sdr_pipeline (sp_id, sector_number) on delete cascade,
 

provider/lpmarket/deal_ingest.go

@@ -0,0 +1,132 @@
+package lpmarket
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-padreader"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/provider/lpseal"
+)
+
+type Ingester interface {
+	AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error)
+}
+
+type PieceIngesterApi interface {
+	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error)
+	StateMinerAllocated(ctx context.Context, a address.Address, key types.TipSetKey) (*bitfield.BitField, error)
+	StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error)
+}
+
+type PieceIngester struct {
+	db  *harmonydb.DB
+	api PieceIngesterApi
+}
+
+func NewPieceIngester(db *harmonydb.DB, api PieceIngesterApi) *PieceIngester {
+	return &PieceIngester{db: db, api: api}
+}
+
+func (p *PieceIngester) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
+	mi, err := p.api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+	if err != nil {
+		return api.SectorOffset{}, err
+	}
+
+	if piece.DealProposal.PieceSize != abi.PaddedPieceSize(mi.SectorSize) {
+		return api.SectorOffset{}, xerrors.Errorf("only full sector pieces supported for now")
+	}
+
+	// check raw size
+	if piece.DealProposal.PieceSize != padreader.PaddedSize(uint64(rawSize)).Padded() {
+		return api.SectorOffset{}, xerrors.Errorf("raw size doesn't match padded piece size")
+	}
+
+	// add initial piece + to a sector
+	nv, err := p.api.StateNetworkVersion(ctx, types.EmptyTSK)
+	if err != nil {
+		return api.SectorOffset{}, xerrors.Errorf("getting network version: %w", err)
+	}
+
+	synth := false // todo synthetic porep config
+	spt, err := miner.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType, synth)
+	if err != nil {
+		return api.SectorOffset{}, xerrors.Errorf("getting seal proof type: %w", err)
+	}
+
+	num, err := lpseal.AllocateSectorNumbers(ctx, p.api, p.db, maddr, 1, func(tx *harmonydb.Tx, numbers []abi.SectorNumber) (bool, error) {
+		if len(numbers) != 1 {
+			return false, xerrors.Errorf("expected one sector number")
+		}
+		n := numbers[0]
+
+		_, err := tx.Exec("insert into sectors_sdr_pipeline (sp_id, sector_number, reg_seal_proof) values ($1, $2, $3)", maddr, n, spt)
+		if err != nil {
+			return false, xerrors.Errorf("inserting into sectors_sdr_pipeline: %w", err)
+		}
+
+		dataHdrJson, err := json.Marshal(header)
+		if err != nil {
+			return false, xerrors.Errorf("json.Marshal(header): %w", err)
+		}
+
+		dealProposalJson, err := json.Marshal(piece.DealProposal)
+		if err != nil {
+			return false, xerrors.Errorf("json.Marshal(piece.DealProposal): %w", err)
+		}
+
+		_, err = tx.Exec(`INSERT INTO sectors_sdr_initial_pieces (sp_id,
+                                        sector_number,
+                                        piece_index,
+                                        
+                                        piece_cid,
+                                        piece_size,
+                                        
+                                        data_url,
+                                        data_headers,
+                                        data_raw_size,
+                                        data_delete_on_finalize,
+                                        
+                                        f05_publish_cid,
+                                        f05_deal_id,
+                                        f05_deal_proposal,
+                                        f05_deal_start_epoch,
+                                        f05_deal_end_epoch) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
+			maddr, n, 0,
+			piece.DealProposal.PieceCID, piece.DealProposal.PieceSize,
+			source.String(), dataHdrJson, rawSize, true,
+			piece.PublishCid, piece.DealID, dealProposalJson, piece.DealSchedule.StartEpoch, piece.DealSchedule.EndEpoch)
+		if err != nil {
+			return false, xerrors.Errorf("inserting into sectors_sdr_initial_pieces: %w", err)
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return api.SectorOffset{}, xerrors.Errorf("allocating sector numbers: %w", err)
+	}
+
+	if len(num) != 1 {
+		return api.SectorOffset{}, xerrors.Errorf("expected one sector number")
+	}
+
+	// After we insert the piece/sector_pipeline entries, the lpseal/poller will take it from here
+
+	return api.SectorOffset{
+		Sector: num[0],
+		Offset: 0,
+	}, nil
+}

provider/lpmarket/fakelm/iface.go

@@ -2,12 +2,15 @@ package fakelm
 
 import (
 	"context"
+	"io"
+
+	"github.com/google/uuid"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
-	"github.com/google/uuid"
-	"io"
 )
 
 // MinimalLMApi is a subset of the LotusMiner API that is exposed by lotus-provider

provider/lpmarket/fakelm/lmimpl.go

@@ -2,15 +2,21 @@ package fakelm
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/provider/lpmarket"
 	"github.com/filecoin-project/lotus/storage/paths"
 	sealing "github.com/filecoin-project/lotus/storage/pipeline"
 	"github.com/filecoin-project/lotus/storage/sealer/storiface"
-	"github.com/google/uuid"
-	"golang.org/x/xerrors"
-	"io"
 )
 
 type LMRPCProvider struct {
@@ -20,6 +26,8 @@ type LMRPCProvider struct {
 	minerID abi.ActorID
 
 	ssize abi.SectorSize
+
+	pi lpmarket.Ingester
 }
 
 func (l *LMRPCProvider) ActorAddress(ctx context.Context) (address.Address, error) {
@@ -259,6 +267,11 @@ func (l *LMRPCProvider) SectorAddPieceToAny(ctx context.Context, size abi.Unpadd
 		return api.SectorOffset{}, xerrors.Errorf("only full-sector pieces are supported")
 	}
 
+	return api.SectorOffset{}, xerrors.Errorf("not supported, use AllocatePieceToSector")
+}
+
+func (l *LMRPCProvider) AllocatePieceToSector(ctx context.Context, maddr address.Address, piece api.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error) {
+	return l.pi.AllocatePieceToSector(ctx, maddr, piece, rawSize, source, header)
 }
 
 var _ MinimalLMApi = &LMRPCProvider{}

provider/lpseal/poller_commit_msg.go

@@ -2,13 +2,16 @@ package lpseal
 
 import (
 	"context"
+
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/exitcode"
+
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
-	"golang.org/x/xerrors"
 )
 
 func (s *SealPoller) pollStartCommitMsg(ctx context.Context, task pollTask) {

provider/lpseal/poller_precommit_msg.go

@@ -2,14 +2,17 @@ package lpseal
 
 import (
 	"context"
+
+	"golang.org/x/xerrors"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/exitcode"
+
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
 	"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
-	"golang.org/x/xerrors"
 )
 
 func (s *SealPoller) pollStartPrecommitMsg(ctx context.Context, task pollTask) {