diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go index 1d7f47bfd..4d8d27a10 100644 --- a/api/mocks/mock_full.go +++ b/api/mocks/mock_full.go @@ -155,23 +155,8 @@ func (mr *MockFullNodeMockRecorder) ChainExport(arg0, arg1, arg2, arg3 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExport", reflect.TypeOf((*MockFullNode)(nil).ChainExport), arg0, arg1, arg2, arg3) } -// ChainExportRange mocks base method. -func (m *MockFullNode) ChainExportRange(arg0 context.Context, arg1, arg2 types.TipSetKey, arg3 *api.ChainExportConfig) (<-chan []byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChainExportRange", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(<-chan []byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ChainExportRange indicates an expected call of ChainExportRange. -func (mr *MockFullNodeMockRecorder) ChainExportRange(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExportRange", reflect.TypeOf((*MockFullNode)(nil).ChainExportRange), arg0, arg1, arg2, arg3) -} - // ChainExportRangeInternal mocks base method. -func (m *MockFullNode) ChainExportRangeInternal(arg0 context.Context, arg1, arg2 types.TipSetKey, arg3 *api.ChainExportConfig) error { +func (m *MockFullNode) ChainExportRangeInternal(arg0 context.Context, arg1, arg2 types.TipSetKey, arg3 api.ChainExportConfig) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ChainExportRangeInternal", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) diff --git a/api/proxy_gen.go b/api/proxy_gen.go index cb3ce775d..e304ed768 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -7,6 +7,15 @@ import ( "encoding/json" "time" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/libp2p/go-libp2p/core/metrics" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -23,6 +32,7 @@ import ( "github.com/filecoin-project/go-state-types/dline" abinetwork "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/proof" + apitypes "github.com/filecoin-project/lotus/api/types" builtinactors "github.com/filecoin-project/lotus/chain/actors/builtin" lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -35,14 +45,6 @@ import ( "github.com/filecoin-project/lotus/storage/sealer/fsutil" "github.com/filecoin-project/lotus/storage/sealer/sealtasks" "github.com/filecoin-project/lotus/storage/sealer/storiface" - "github.com/google/uuid" - "github.com/ipfs/go-cid" - blocks "github.com/ipfs/go-libipfs/blocks" - "github.com/libp2p/go-libp2p/core/metrics" - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" - "golang.org/x/xerrors" ) var ErrNotSupported = xerrors.New("method not supported") diff --git a/api/v0api/proxy_gen.go b/api/v0api/proxy_gen.go index ebfcb1fe9..5fa0d949c 100644 --- a/api/v0api/proxy_gen.go +++ b/api/v0api/proxy_gen.go @@ -5,6 +5,11 @@ package v0api import ( "context" + "github.com/ipfs/go-cid" + blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/libp2p/go-libp2p/core/peer" + "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-bitfield" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -17,6 +22,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/dline" abinetwork "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/api" apitypes "github.com/filecoin-project/lotus/api/types" lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -24,10 +30,6 @@ import ( marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo/imports" - "github.com/ipfs/go-cid" - blocks "github.com/ipfs/go-libipfs/blocks" - "github.com/libp2p/go-libp2p/core/peer" - "golang.org/x/xerrors" ) var ErrNotSupported = xerrors.New("method not supported") diff --git a/api/v0api/v0mocks/mock_full.go b/api/v0api/v0mocks/mock_full.go index c99dbae0e..619f19d35 100644 --- a/api/v0api/v0mocks/mock_full.go +++ b/api/v0api/v0mocks/mock_full.go @@ -141,35 +141,6 @@ func (mr *MockFullNodeMockRecorder) ChainExport(arg0, arg1, arg2, arg3 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExport", reflect.TypeOf((*MockFullNode)(nil).ChainExport), arg0, arg1, arg2, arg3) } -// ChainExportRange mocks base method. -func (m *MockFullNode) ChainExportRange(arg0 context.Context, arg1, arg2 types.TipSetKey, arg3 *api.ChainExportConfig) (<-chan []byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChainExportRange", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(<-chan []byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ChainExportRange indicates an expected call of ChainExportRange. -func (mr *MockFullNodeMockRecorder) ChainExportRange(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExportRange", reflect.TypeOf((*MockFullNode)(nil).ChainExportRange), arg0, arg1, arg2, arg3) -} - -// ChainExportRangeInternal mocks base method. -func (m *MockFullNode) ChainExportRangeInternal(arg0 context.Context, arg1, arg2 types.TipSetKey, arg3 *api.ChainExportConfig) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChainExportRangeInternal", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// ChainExportRangeInternal indicates an expected call of ChainExportRangeInternal. -func (mr *MockFullNodeMockRecorder) ChainExportRangeInternal(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExportRangeInternal", reflect.TypeOf((*MockFullNode)(nil).ChainExportRangeInternal), arg0, arg1, arg2, arg3) -} - // ChainGetBlock mocks base method. func (m *MockFullNode) ChainGetBlock(arg0 context.Context, arg1 cid.Cid) (*types.BlockHeader, error) { m.ctrl.T.Helper() diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 952af5430..c575896e7 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/gateway.json.gz b/build/openrpc/gateway.json.gz index 3f6c25090..84fd0cf60 100644 Binary files a/build/openrpc/gateway.json.gz and b/build/openrpc/gateway.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index c08797817..b809e1e3b 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 55084d176..9751906e1 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/documentation/en/api-v0-methods.md b/documentation/en/api-v0-methods.md index 896c3ca1e..fe639b2f3 100644 --- a/documentation/en/api-v0-methods.md +++ b/documentation/en/api-v0-methods.md @@ -13,8 +13,6 @@ * [Chain](#Chain) * [ChainDeleteObj](#ChainDeleteObj) * [ChainExport](#ChainExport) - * [ChainExportRange](#ChainExportRange) - * [ChainExportRangeInternal](#ChainExportRangeInternal) * [ChainGetBlock](#ChainGetBlock) * [ChainGetBlockMessages](#ChainGetBlockMessages) * [ChainGetGenesis](#ChainGetGenesis) @@ -423,80 +421,6 @@ Inputs: Response: `"Ynl0ZSBhcnJheQ=="` -### ChainExportRange - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "WriteBufferSize": 123, - "Workers": 9, - "CacheSize": 123, - "IncludeMessages": true, - "IncludeReceipts": true, - "IncludeStateRoots": true - } -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - -### ChainExportRangeInternal - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "WriteBufferSize": 123, - "Workers": 9, - "CacheSize": 123, - "IncludeMessages": true, - "IncludeReceipts": true, - "IncludeStateRoots": true - } -] -``` - -Response: `{}` - ### ChainGetBlock ChainGetBlock returns the block specified by the given CID. diff --git a/documentation/en/api-v1-unstable-methods.md b/documentation/en/api-v1-unstable-methods.md index 37ff77a00..10153303c 100644 --- a/documentation/en/api-v1-unstable-methods.md +++ b/documentation/en/api-v1-unstable-methods.md @@ -13,7 +13,6 @@ * [ChainCheckBlockstore](#ChainCheckBlockstore) * [ChainDeleteObj](#ChainDeleteObj) * [ChainExport](#ChainExport) - * [ChainExportRange](#ChainExportRange) * [ChainExportRangeInternal](#ChainExportRangeInternal) * [ChainGetBlock](#ChainGetBlock) * [ChainGetBlockMessages](#ChainGetBlockMessages) @@ -474,47 +473,18 @@ Inputs: Response: `"Ynl0ZSBhcnJheQ=="` -### ChainExportRange - - -Perms: read - -Inputs: -```json -[ - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - [ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - { - "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" - } - ], - { - "WriteBufferSize": 123, - "Workers": 9, - "CacheSize": 123, - "IncludeMessages": true, - "IncludeReceipts": true, - "IncludeStateRoots": true - } -] -``` - -Response: `"Ynl0ZSBhcnJheQ=="` - ### ChainExportRangeInternal +ChainExportRangeInternal triggers the export of a chain +CAR-snapshot directly to disk. It is similar to ChainExport, +except, depending on options, the snapshot can include receipts, +messages and stateroots for the length between the specified head +and tail, thus producing "archival-grade" snapshots that include +all the on-chain data. The header chain is included back to +genesis and these snapshots can be used to initialize Filecoin +nodes. -Perms: read +Perms: admin Inputs: ```json @@ -537,8 +507,7 @@ Inputs: ], { "WriteBufferSize": 123, - "Workers": 9, - "CacheSize": 123, + "NumWorkers": 123, "IncludeMessages": true, "IncludeReceipts": true, "IncludeStateRoots": true diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 53f8e4e8a..7800503aa 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -2110,6 +2110,7 @@ COMMANDS: get Get chain DAG node by path bisect bisect chain for an event export export chain to a car file + export-range export chain to a car file slash-consensus Report consensus fault gas-price Estimate gas prices inspect-usage Inspect block space usage of a given tipset @@ -2291,6 +2292,25 @@ OPTIONS: ``` +### lotus chain export-range +``` +NAME: + lotus chain export-range - export chain to a car file + +USAGE: + lotus chain export-range [command options] [arguments...] + +OPTIONS: + --head value specify tipset to start the export from (higher epoch) (default: "@head") + --messages specify if messages should be include (default: false) + --receipts specify if receipts should be include (default: false) + --stateroots specify if stateroots should be include (default: false) + --tail value specify tipset to end the export at (lower epoch) (default: "@tail") + --workers value specify the number of workers (default: 1) + --write-buffer value specify write buffer size (default: 1048576) + +``` + ### lotus chain slash-consensus ``` NAME: diff --git a/go.mod b/go.mod index 720f2f727..17492d0e6 100644 --- a/go.mod +++ b/go.mod @@ -153,6 +153,7 @@ require ( go.opentelemetry.io/otel/bridge/opencensus v0.33.0 go.opentelemetry.io/otel/exporters/jaeger v1.2.0 go.opentelemetry.io/otel/sdk v1.11.1 + go.uber.org/atomic v1.10.0 go.uber.org/fx v1.15.0 go.uber.org/multierr v1.8.0 go.uber.org/zap v1.23.0 @@ -334,7 +335,6 @@ require ( go.opentelemetry.io/otel/metric v0.33.0 // indirect go.opentelemetry.io/otel/sdk/metric v0.33.0 // indirect go.opentelemetry.io/otel/trace v1.11.1 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.12.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect