have a typed importID

This commit is contained in:
aarshkshah1992 2021-07-07 12:02:48 +05:30
parent 2b5164c956
commit ba236fe258
14 changed files with 43 additions and 2333 deletions

View File

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"time" "time"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peer"
@ -319,7 +320,7 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore. // ClientImport imports file under the specified path into filestore.
ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin ClientImport(ctx context.Context, ref FileRef) (*ImportRes, error) //perm:admin
// ClientRemoveImport removes file import // ClientRemoveImport removes file import
ClientRemoveImport(ctx context.Context, importID uint64) error //perm:admin ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
// ClientStartDeal proposes a deal with a miner. // ClientStartDeal proposes a deal with a miner.
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error) //perm:admin
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking. // ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.
@ -711,11 +712,11 @@ type MinerSectors struct {
type ImportRes struct { type ImportRes struct {
Root cid.Cid Root cid.Cid
ImportID uint64 ImportID importmgr.ImportID
} }
type Import struct { type Import struct {
Key uint64 Key importmgr.ImportID
Err string Err string
Root *cid.Cid Root *cid.Cid

View File

@ -13,6 +13,7 @@ import (
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-bitfield"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore" "github.com/ipfs/go-filestore"
@ -87,7 +88,7 @@ func init() {
addExample(pid) addExample(pid)
addExample(&pid) addExample(&pid)
storeIDExample := uint64(50) storeIDExample := importmgr.ImportID(50)
addExample(bitfield.NewFromSet([]uint64{5})) addExample(bitfield.NewFromSet([]uint64{5}))
addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1) addExample(abi.RegisteredSealProof_StackedDrg32GiBV1_1)

View File

@ -25,6 +25,7 @@ import (
types "github.com/filecoin-project/lotus/chain/types" types "github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers" marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock" gomock "github.com/golang/mock/gomock"
@ -715,7 +716,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
} }
// ClientRemoveImport mocks base method. // ClientRemoveImport mocks base method.
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 uint64) error { func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1) ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
ret0, _ := ret[0].(error) ret0, _ := ret[0].(error)

View File

@ -28,6 +28,7 @@ import (
"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
marketevents "github.com/filecoin-project/lotus/markets/loggers" marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/specs-storage/storage"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
@ -196,7 +197,7 @@ type FullNodeStruct struct {
ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"` ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"`
ClientRemoveImport func(p0 context.Context, p1 uint64) error `perm:"admin"` ClientRemoveImport func(p0 context.Context, p1 importmgr.ImportID) error `perm:"admin"`
ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"` ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
@ -1380,11 +1381,11 @@ func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address
return nil, xerrors.New("method not supported") return nil, xerrors.New("method not supported")
} }
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 uint64) error { func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return s.Internal.ClientRemoveImport(p0, p1) return s.Internal.ClientRemoveImport(p0, p1)
} }
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 uint64) error { func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return xerrors.New("method not supported") return xerrors.New("method not supported")
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/dline"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peer"
@ -300,7 +301,7 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore. // ClientImport imports file under the specified path into filestore.
ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin ClientImport(ctx context.Context, ref api.FileRef) (*api.ImportRes, error) //perm:admin
// ClientRemoveImport removes file import // ClientRemoveImport removes file import
ClientRemoveImport(ctx context.Context, importID uint64) error //perm:admin ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error //perm:admin
// ClientStartDeal proposes a deal with a miner. // ClientStartDeal proposes a deal with a miner.
ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin ClientStartDeal(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) //perm:admin
// ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking. // ClientStatelessDeal fire-and-forget-proposes an offline deal to a miner without subsequent tracking.

View File

@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers" marketevents "github.com/filecoin-project/lotus/markets/loggers"
"github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors" "golang.org/x/xerrors"
@ -114,7 +115,7 @@ type FullNodeStruct struct {
ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"` ClientQueryAsk func(p0 context.Context, p1 peer.ID, p2 address.Address) (*storagemarket.StorageAsk, error) `perm:"read"`
ClientRemoveImport func(p0 context.Context, p1 uint64) error `perm:"admin"` ClientRemoveImport func(p0 context.Context, p1 importmgr.ImportID) error `perm:"admin"`
ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"` ClientRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"`
@ -793,11 +794,11 @@ func (s *FullNodeStub) ClientQueryAsk(p0 context.Context, p1 peer.ID, p2 address
return nil, xerrors.New("method not supported") return nil, xerrors.New("method not supported")
} }
func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 uint64) error { func (s *FullNodeStruct) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return s.Internal.ClientRemoveImport(p0, p1) return s.Internal.ClientRemoveImport(p0, p1)
} }
func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 uint64) error { func (s *FullNodeStub) ClientRemoveImport(p0 context.Context, p1 importmgr.ImportID) error {
return xerrors.New("method not supported") return xerrors.New("method not supported")
} }

View File

@ -25,6 +25,7 @@ import (
types "github.com/filecoin-project/lotus/chain/types" types "github.com/filecoin-project/lotus/chain/types"
marketevents "github.com/filecoin-project/lotus/markets/loggers" marketevents "github.com/filecoin-project/lotus/markets/loggers"
dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
importmgr "github.com/filecoin-project/lotus/node/repo/importmgr"
miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner"
paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" paych "github.com/filecoin-project/specs-actors/actors/builtin/paych"
gomock "github.com/golang/mock/gomock" gomock "github.com/golang/mock/gomock"
@ -715,7 +716,7 @@ func (mr *MockFullNodeMockRecorder) ClientQueryAsk(arg0, arg1, arg2 interface{})
} }
// ClientRemoveImport mocks base method. // ClientRemoveImport mocks base method.
func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 uint64) error { func (m *MockFullNode) ClientRemoveImport(arg0 context.Context, arg1 importmgr.ImportID) error {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1) ret := m.ctrl.Call(m, "ClientRemoveImport", arg0, arg1)
ret0, _ := ret[0].(error) ret0, _ := ret[0].(error)

View File

@ -25,6 +25,7 @@ import (
"github.com/fatih/color" "github.com/fatih/color"
datatransfer "github.com/filecoin-project/go-data-transfer" datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil/cidenc" "github.com/ipfs/go-cidutil/cidenc"
"github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peer"
@ -184,7 +185,7 @@ var clientDropCmd = &cli.Command{
} }
for _, id := range ids { for _, id := range ids {
if err := api.ClientRemoveImport(ctx, id); err != nil { if err := api.ClientRemoveImport(ctx, importmgr.ImportID(id)); err != nil {
return xerrors.Errorf("removing import %d: %w", id, err) return xerrors.Errorf("removing import %d: %w", id, err)
} }
} }

File diff suppressed because it is too large Load Diff

View File

@ -1286,7 +1286,7 @@ Response:
"Root": { "Root": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
}, },
"ImportID": 42 "ImportID": 50
} }
``` ```
@ -1406,7 +1406,7 @@ Perms: admin
Inputs: Inputs:
```json ```json
[ [
42 50
] ]
``` ```

View File

@ -1288,7 +1288,7 @@ Response:
"Root": { "Root": {
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
}, },
"ImportID": 42 "ImportID": 50
} }
``` ```
@ -1408,7 +1408,7 @@ Perms: admin
Inputs: Inputs:
```json ```json
[ [
42 50
] ]
``` ```

View File

@ -513,7 +513,7 @@ func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (res *api.Impor
}, nil }, nil
} }
func (a *API) ClientRemoveImport(ctx context.Context, importID uint64) error { func (a *API) ClientRemoveImport(ctx context.Context, importID importmgr.ImportID) error {
info, err := a.imgr().Info(importID) info, err := a.imgr().Info(importID)
if err != nil { if err != nil {
return xerrors.Errorf("failed to fetch import info: %w", err) return xerrors.Errorf("failed to fetch import info: %w", err)
@ -529,7 +529,7 @@ func (a *API) ClientRemoveImport(ctx context.Context, importID uint64) error {
func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, error) { func (a *API) ClientImportLocal(ctx context.Context, r io.Reader) (cid.Cid, error) {
// write payload to temp file // write payload to temp file
tmpPath, err := a.imgr().NewTempFile(rand.Uint64()) tmpPath, err := a.imgr().NewTempFile(importmgr.ImportID(rand.Uint64()))
if err != nil { if err != nil {
return cid.Undef, err return cid.Undef, err
} }
@ -1037,7 +1037,7 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
} }
func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error { func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath string) error {
id := rand.Uint64() id := importmgr.ImportID(rand.Uint64())
tmpCARv2File, err := a.imgr().NewTempFile(id) tmpCARv2File, err := a.imgr().NewTempFile(id)
if err != nil { if err != nil {
return xerrors.Errorf("failed to create temp file: %w", err) return xerrors.Errorf("failed to create temp file: %w", err)

View File

@ -7,6 +7,7 @@ import (
"os" "os"
"github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice" "github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/ipfs/go-cidutil" "github.com/ipfs/go-cidutil"
@ -24,7 +25,7 @@ import (
) )
// importNormalFileToCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file at the given output path. // importNormalFileToCARv2 transforms the client's "normal file" to a Unixfs IPLD DAG and writes out the DAG to a CARv2 file at the given output path.
func (a *API) importNormalFileToCARv2(ctx context.Context, importID uint64, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) { func (a *API) importNormalFileToCARv2(ctx context.Context, importID importmgr.ImportID, inputFilePath string, outputCARv2Path string) (c cid.Cid, finalErr error) {
// TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger. // TODO: We've currently put in a hack to create the Unixfs DAG as a CARv2 without using Badger.
// We first transform the Unixfs DAG to a rootless CARv2 file as CARv2 doesen't allow streaming writes without specifying the root upfront and we // We first transform the Unixfs DAG to a rootless CARv2 file as CARv2 doesen't allow streaming writes without specifying the root upfront and we

View File

@ -15,6 +15,8 @@ import (
"github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/namespace"
) )
type ImportID uint64
type Mgr struct { type Mgr struct {
ds datastore.Batching ds datastore.Batching
repoPath string repoPath string
@ -42,7 +44,7 @@ type StoreMeta struct {
Labels map[string]string Labels map[string]string
} }
func (m *Mgr) NewStore() (uint64, error) { func (m *Mgr) NewStore() (ImportID, error) {
id := m.counter.Next() id := m.counter.Next()
meta, err := json.Marshal(&StoreMeta{Labels: map[string]string{ meta, err := json.Marshal(&StoreMeta{Labels: map[string]string{
@ -53,10 +55,10 @@ func (m *Mgr) NewStore() (uint64, error) {
} }
err = m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta) err = m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta)
return id, err return ImportID(id), err
} }
func (m *Mgr) AddLabel(id uint64, key, value string) error { // source, file path, data CID.. func (m *Mgr) AddLabel(id ImportID, key, value string) error { // source, file path, data CID..
meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id))) meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
if err != nil { if err != nil {
return xerrors.Errorf("getting metadata form datastore: %w", err) return xerrors.Errorf("getting metadata form datastore: %w", err)
@ -77,8 +79,8 @@ func (m *Mgr) AddLabel(id uint64, key, value string) error { // source, file pat
return m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta) return m.ds.Put(datastore.NewKey(fmt.Sprintf("%d", id)), meta)
} }
func (m *Mgr) List() ([]uint64, error) { func (m *Mgr) List() ([]ImportID, error) {
var keys []uint64 var keys []ImportID
qres, err := m.ds.Query(query.Query{KeysOnly: true}) qres, err := m.ds.Query(query.Query{KeysOnly: true})
if err != nil { if err != nil {
@ -96,13 +98,13 @@ func (m *Mgr) List() ([]uint64, error) {
if err != nil { if err != nil {
return nil, xerrors.Errorf("failed to parse key %s to uint64, err=%w", r.Key, err) return nil, xerrors.Errorf("failed to parse key %s to uint64, err=%w", r.Key, err)
} }
keys = append(keys, id) keys = append(keys, ImportID(id))
} }
return keys, nil return keys, nil
} }
func (m *Mgr) Info(id uint64) (*StoreMeta, error) { func (m *Mgr) Info(id ImportID) (*StoreMeta, error) {
meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id))) meta, err := m.ds.Get(datastore.NewKey(fmt.Sprintf("%d", id)))
if err != nil { if err != nil {
return nil, xerrors.Errorf("getting metadata form datastore: %w", err) return nil, xerrors.Errorf("getting metadata form datastore: %w", err)
@ -116,7 +118,7 @@ func (m *Mgr) Info(id uint64) (*StoreMeta, error) {
return &sm, nil return &sm, nil
} }
func (m *Mgr) Remove(id uint64) error { func (m *Mgr) Remove(id ImportID) error {
if err := m.ds.Delete(datastore.NewKey(fmt.Sprintf("%d", id))); err != nil { if err := m.ds.Delete(datastore.NewKey(fmt.Sprintf("%d", id))); err != nil {
return xerrors.Errorf("removing import metadata: %w", err) return xerrors.Errorf("removing import metadata: %w", err)
} }
@ -150,8 +152,8 @@ func (m *Mgr) CARV2FilePathFor(dagRoot cid.Cid) (string, error) {
return "", nil return "", nil
} }
func (m *Mgr) NewTempFile(id uint64) (string, error) { func (m *Mgr) NewTempFile(importID ImportID) (string, error) {
file, err := ioutil.TempFile(m.repoPath, fmt.Sprintf("%d", id)) file, err := ioutil.TempFile(m.repoPath, fmt.Sprintf("%d", importID))
if err != nil { if err != nil {
return "", xerrors.Errorf("failed to create temp file: %w", err) return "", xerrors.Errorf("failed to create temp file: %w", err)
} }