From 0800e6e5a7c015b89057d15616745330ca64f79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 28 Feb 2024 21:50:12 +0100 Subject: [PATCH] make gen --- build/openrpc/miner.json | 10 ++++++---- cmd/lotus-provider/tasks/tasks.go | 6 +++--- documentation/en/api-v0-methods-miner.md | 2 ++ .../en/default-lotus-provider-config.toml | 12 ++++++++++++ .../harmonydb/sql/20240228-piece-park.sql | 16 +++++++++++----- node/config/doc_gen.go | 16 ++++++++++++++++ provider/lpffi/piece_funcs.go | 6 ++++-- provider/lppiece/task_park_piece.go | 13 ++++++++----- storage/paths/http_handler.go | 3 ++- 9 files changed, 64 insertions(+), 20 deletions(-) diff --git a/build/openrpc/miner.json b/build/openrpc/miner.json index 0686987c8..2b0887ae2 100644 --- a/build/openrpc/miner.json +++ b/build/openrpc/miner.json @@ -9711,6 +9711,7 @@ 0, 1, 0, + 0, 0 ], "Read": [ @@ -9718,6 +9719,7 @@ 3, 0, 0, + 0, 0 ] } @@ -9736,8 +9738,8 @@ "title": "number", "type": "number" }, - "maxItems": 5, - "minItems": 5, + "maxItems": 6, + "minItems": 6, "type": "array" }, "Sector": { @@ -9760,8 +9762,8 @@ "title": "number", "type": "number" }, - "maxItems": 5, - "minItems": 5, + "maxItems": 6, + "minItems": 6, "type": "array" } }, diff --git a/cmd/lotus-provider/tasks/tasks.go b/cmd/lotus-provider/tasks/tasks.go index 11c12216b..71e794545 100644 --- a/cmd/lotus-provider/tasks/tasks.go +++ b/cmd/lotus-provider/tasks/tasks.go @@ -3,9 +3,6 @@ package tasks import ( "context" - "github.com/filecoin-project/lotus/lib/lazy" - "github.com/filecoin-project/lotus/lib/must" - "github.com/filecoin-project/lotus/provider/lppiece" logging "github.com/ipfs/go-log/v2" "github.com/samber/lo" @@ -13,11 +10,14 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-provider/deps" "github.com/filecoin-project/lotus/lib/harmony/harmonytask" + "github.com/filecoin-project/lotus/lib/lazy" + "github.com/filecoin-project/lotus/lib/must" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/provider" "github.com/filecoin-project/lotus/provider/chainsched" "github.com/filecoin-project/lotus/provider/lpffi" "github.com/filecoin-project/lotus/provider/lpmessage" + "github.com/filecoin-project/lotus/provider/lppiece" "github.com/filecoin-project/lotus/provider/lpseal" "github.com/filecoin-project/lotus/provider/lpwinning" ) diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index b133930bc..dd2f511db 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -3954,6 +3954,7 @@ Response: 0, 1, 0, + 0, 0 ], "Read": [ @@ -3961,6 +3962,7 @@ Response: 3, 0, 0, + 0, 0 ] } diff --git a/documentation/en/default-lotus-provider-config.toml b/documentation/en/default-lotus-provider-config.toml index dd9921769..83a7c243d 100644 --- a/documentation/en/default-lotus-provider-config.toml +++ b/documentation/en/default-lotus-provider-config.toml @@ -25,6 +25,18 @@ # type: int #WinningPostMaxTasks = 0 + # EnableParkPiece enables the "piece parking" task to run on this node. This task is responsible for fetching + # pieces from the network and storing them in the storage subsystem until sectors are sealed. This task is + # only applicable when integrating with boost, and should be enabled on nodes which will hold deal data + # from boost until sectors containing the related pieces have the TreeD/TreeR constructed. + # Note that future Curio implementations will have a separate task type for fetching pieces from the internet. + # + # type: bool + #EnableParkPiece = false + + # type: int + #ParkPieceMaxTasks = 0 + # EnableSealSDR enables SDR tasks to run. SDR is the long sequential computation # creating 11 layer files in sector cache directory. # diff --git a/lib/harmony/harmonydb/sql/20240228-piece-park.sql b/lib/harmony/harmonydb/sql/20240228-piece-park.sql index c58ecdafe..a1bf0efa9 100644 --- a/lib/harmony/harmonydb/sql/20240228-piece-park.sql +++ b/lib/harmony/harmonydb/sql/20240228-piece-park.sql @@ -1,17 +1,23 @@ create table parked_pieces ( - id serial primary key, + id bigserial primary key, created_at timestamp default current_timestamp, piece_cid text not null, piece_padded_size bigint not null, piece_raw_size text not null, - + complete boolean not null default false ); +/* + * This table is used to keep track of the references to the parked pieces + * so that we can delete them when they are no longer needed. + * + * All references into the parked_pieces table should be done through this table. + */ create table parked_piece_refs ( - ref_id serial primary key, - piece_id int not null, + ref_id bigserial primary key, + piece_id bigint not null, foreign key (piece_id) references parked_pieces(id) on delete cascade ); @@ -21,7 +27,7 @@ create table park_piece_tasks ( constraint park_piece_tasks_pk primary key, - piece_ref_id int not null, + piece_ref_id bigint not null, data_url text not null, data_headers jsonb not null default '{}', diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index f28c5abd8..c49dad3ac 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1033,6 +1033,22 @@ documentation.`, Comment: ``, }, + { + Name: "EnableParkPiece", + Type: "bool", + + Comment: `EnableParkPiece enables the "piece parking" task to run on this node. This task is responsible for fetching +pieces from the network and storing them in the storage subsystem until sectors are sealed. This task is +only applicable when integrating with boost, and should be enabled on nodes which will hold deal data +from boost until sectors containing the related pieces have the TreeD/TreeR constructed. +Note that future Curio implementations will have a separate task type for fetching pieces from the internet.`, + }, + { + Name: "ParkPieceMaxTasks", + Type: "int", + + Comment: ``, + }, { Name: "EnableSealSDR", Type: "bool", diff --git a/provider/lpffi/piece_funcs.go b/provider/lpffi/piece_funcs.go index 4e2816786..2ee75151b 100644 --- a/provider/lpffi/piece_funcs.go +++ b/provider/lpffi/piece_funcs.go @@ -2,11 +2,13 @@ package lpffi import ( "context" - "github.com/filecoin-project/lotus/storage/sealer/storiface" - "golang.org/x/xerrors" "io" "os" "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func (sb *SealCalls) WritePiece(ctx context.Context, pieceID storiface.PieceNumber, size int64, data io.Reader) error { diff --git a/provider/lppiece/task_park_piece.go b/provider/lppiece/task_park_piece.go index 370775803..2ddcd7617 100644 --- a/provider/lppiece/task_park_piece.go +++ b/provider/lppiece/task_park_piece.go @@ -3,7 +3,15 @@ package lppiece import ( "context" "encoding/json" + "net/http" + "time" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/lib/harmony/harmonytask" "github.com/filecoin-project/lotus/lib/harmony/resources" @@ -11,11 +19,6 @@ import ( "github.com/filecoin-project/lotus/provider/lpffi" "github.com/filecoin-project/lotus/provider/lpseal" "github.com/filecoin-project/lotus/storage/sealer/storiface" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - "golang.org/x/xerrors" - "net/http" - "time" ) var log = logging.Logger("lppiece") diff --git a/storage/paths/http_handler.go b/storage/paths/http_handler.go index 92f4162db..c828f6006 100644 --- a/storage/paths/http_handler.go +++ b/storage/paths/http_handler.go @@ -9,11 +9,12 @@ import ( "strconv" "time" - "github.com/filecoin-project/go-state-types/abi" "github.com/gorilla/mux" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/storage/sealer/partialfile" "github.com/filecoin-project/lotus/storage/sealer/storiface" "github.com/filecoin-project/lotus/storage/sealer/tarutil"