cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

have a typed importID

Browse Source
This commit is contained in:
aarshkshah1992 2021-07-07 12:02:48 +05:30
parent 2b5164c956
commit ba236fe258
14 changed files with 43 additions and 2333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
api/api_full.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"time"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
@ -319,7 +320,7 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore.
ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin
// ClientRemoveImport removes file import
ClientRemoveImport(ctx context.Context, importID uint64) error //perm:admin
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
// ClientStartDeal proposes a deal with a miner.
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
@ -711,11 +712,11 @@ type MinerSectors struct {
type ImportRes struct {
Root cid.Cid
ImportID uint64
ImportID importmgr.ImportID
}
type Import struct {
Key uint64
Key importmgr.ImportID
Err string
Root *cid.Cid

3
api/docgen/docgen.go
View File

@ -13,6 +13,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore"
@ -87,7 +88,7 @@ func init() {
addExample(pid)
addExample(&pid)
storeIDExample := uint64(50)
storeIDExample := importmgr.ImportID(50)
addExample(bitfield.NewFromSet([]uint64{5}))
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)

3
api/mocks/mock_full.go
View File

@ -25,6 +25,7 @@ import (
types "github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock"
@ -715,7 +716,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
}
// ClientRemoveImport mocks base method.
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 uint64) error {
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
ret0, _ := ret[0].(error)

7
api/proxy_gen.go
View File

@ -28,6 +28,7 @@ import (
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/filecoin-project/specs-storage/storage"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
@ -196,7 +197,7 @@ type FullNodeStruct struct {
ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"`
ClientRemoveImport func(p0 context.Context, p1 uint64) error `perm:"admin"`
ClientRemoveImport func(p0 context.Context, p1 importmgr.ImportID) error `perm:"admin"`
ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
@ -1380,11 +1381,11 @@ func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address
return nil, xerrors.New("method not supported")
}
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 uint64) error {
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return s.Internal.ClientRemoveImport(p0, p1)
}
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 uint64) error {
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return xerrors.New("method not supported")
}

3
api/v0api/full.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
@ -300,7 +301,7 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore.
ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
// ClientRemoveImport removes file import
ClientRemoveImport(ctx context.Context, importID uint64) error //perm:admin
ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
// ClientStartDeal proposes a deal with a miner.
ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.

7
api/v0api/proxy_gen.go
View File

@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
@ -114,7 +115,7 @@ type FullNodeStruct struct {
ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"`
ClientRemoveImport func(p0 context.Context, p1 uint64) error `perm:"admin"`
ClientRemoveImport func(p0 context.Context, p1 importmgr.ImportID) error `perm:"admin"`
ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
@ -793,11 +794,11 @@ func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address
return nil, xerrors.New("method not supported")
}
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 uint64) error {
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return s.Internal.ClientRemoveImport(p0, p1)
}
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 uint64) error {
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return xerrors.New("method not supported")
}

3
api/v0api/v0mocks/mock_full.go
View File

@ -25,6 +25,7 @@ import (
types "github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock"
@ -715,7 +716,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
}
// ClientRemoveImport mocks base method.
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 uint64) error {
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
ret0, _ := ret[0].(error)

3
cli/client.go
View File

@ -25,6 +25,7 @@ import (
"github.com/fatih/color"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil/cidenc"
"github.com/libp2p/go-libp2p-core/peer"
@ -184,7 +185,7 @@ var clientDropCmd = &cli.Command{
}
for _, id := range ids {
if err := api.ClientRemoveImport(ctx, id); err != nil {
if err := api.ClientRemoveImport(ctx, importmgr.ImportID(id)); err != nil {
return xerrors.Errorf("removing import %d: %w", id, err)
}
}

2301
documentation/en/api-v0-methods-miner.md
View File

File diff suppressed because it is too large Load Diff

4
documentation/en/api-v0-methods.md
View File

@ -1286,7 +1286,7 @@ Response:
"Root": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
"ImportID": 42
"ImportID": 50
}
```
@ -1406,7 +1406,7 @@ Perms: admin
Inputs:
```json
[
42
50
]
```

4
documentation/en/api-v1-unstable-methods.md
View File

@ -1288,7 +1288,7 @@ Response:
"Root": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
"ImportID": 42
"ImportID": 50
}
```
@ -1408,7 +1408,7 @@ Perms: admin
Inputs:
```json
[
42
50
]
```

6
node/impl/client/client.go
View File

@ -513,7 +513,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
}, nil
}
func (a *API) ClientRemoveImport(ctx context.Context, importID uint64) error {
func (a *API) ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error {
info, err := a.imgr().Info(importID)
if err != nil {
return xerrors.Errorf("failed to fetch import info: %w", err)
@ -529,7 +529,7 @@ func (a *API) ClientRemoveImport(ctx context.Context, importID uint64) error {
func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, error) {
// write payload to temp file
tmpPath, err := a.imgr().NewTempFile(rand.Uint64())
tmpPath, err := a.imgr().NewTempFile(importmgr.ImportID(rand.Uint64()))
if err != nil {
return cid.Undef, err
}
@ -1037,7 +1037,7 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
}
func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error {
id := rand.Uint64()
id := importmgr.ImportID(rand.Uint64())
tmpCARv2File, err := a.imgr().NewTempFile(id)
if err != nil {
return xerrors.Errorf("failed to create temp file: %w", err)

3
node/impl/client/import.go
View File

@ -7,6 +7,7 @@ import (
"os"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil"
@ -24,7 +25,7 @@ import (
)
// importNormalFileToCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file at the given output path.
func (a *API) importNormalFileToCARv2(ctx context.Context, importID uint64, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
func (a *API) importNormalFileToCARv2(ctx context.Context, importID importmgr.ImportID, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
// TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger.
// We first transform the Unixfs DAG to a rootless CARv2 file as CARv2 doesen't allow streaming writes without specifying the root upfront and we

22
node/repo/importmgr/mgr.go
View File

@ -15,6 +15,8 @@ import (
"github.com/ipfs/go-datastore/namespace"
)
type ImportID uint64
type Mgr struct {
ds datastore.Batching
repoPath string
@ -42,7 +44,7 @@ type StoreMeta struct {
Labels map[string]string
}
func (m *Mgr) NewStore() (uint64, error) {
func (m *Mgr) NewStore() (ImportID, error) {
id := m.counter.Next()
meta, err := json.Marshal(&StoreMeta{Labels: map[string]string{
@ -53,10 +55,10 @@ func (m *Mgr) NewStore() (uint64, error) {
}
err = m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta)
return id, err
return ImportID(id), err
}
func (m *Mgr) AddLabel(id uint64, key, value string) error { // source, file path, data CID..
func (m *Mgr) AddLabel(id ImportID, key, value string) error { // source, file path, data CID..
meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
if err != nil {
return xerrors.Errorf("getting metadata form datastore: %w", err)
@ -77,8 +79,8 @@ func (m *Mgr) AddLabel(id uint64, key, value string) error { // source, file pat
return m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta)
}
func (m *Mgr) List() ([]uint64, error) {
var keys []uint64
func (m *Mgr) List() ([]ImportID, error) {
var keys []ImportID
qres, err := m.ds.Query(query.Query{KeysOnly: true})
if err != nil {
@ -96,13 +98,13 @@ func (m *Mgr) List() ([]uint64, error) {
if err != nil {
return nil, xerrors.Errorf("failed to parse key %s to uint64, err=%w", r.Key, err)
}
keys = append(keys, id)
keys = append(keys, ImportID(id))
}
return keys, nil
}
func (m *Mgr) Info(id uint64) (*StoreMeta, error) {
func (m *Mgr) Info(id ImportID) (*StoreMeta, error) {
meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
if err != nil {
return nil, xerrors.Errorf("getting metadata form datastore: %w", err)
@ -116,7 +118,7 @@ func (m *Mgr) Info(id uint64) (*StoreMeta, error) {
return &sm, nil
}
func (m *Mgr) Remove(id uint64) error {
func (m *Mgr) Remove(id ImportID) error {
if err := m.ds.Delete(datastore.NewKey(fmt.Sprintf("%d", id))); err != nil {
return xerrors.Errorf("removing import metadata: %w", err)
}
@ -150,8 +152,8 @@ func (m *Mgr) CARV2FilePathFor(dagRoot cid.Cid) (string, error) {
return "", nil
}
func (m *Mgr) NewTempFile(id uint64) (string, error) {
file, err := ioutil.TempFile(m.repoPath, fmt.Sprintf("%d", id))
func (m *Mgr) NewTempFile(importID ImportID) (string, error) {
file, err := ioutil.TempFile(m.repoPath, fmt.Sprintf("%d", importID))
if err != nil {
return "", xerrors.Errorf("failed to create temp file: %w", err)
}