cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

feat(client): add car import/export option

Browse Source 
adds option on client file import and client retrieval to read/write to CAR file
This commit is contained in:
hannahhoward 2020-03-02 20:13:08 -08:00
parent 3a923e8aae
commit 9f5f70a93f
6 changed files with 80 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/api_full.go
View File

@ -89,13 +89,13 @@ type FullNode interface {
// Other
// ClientImport imports file under the specified path into filestore
ClientImport(ctx context.Context, path string) (cid.Cid, error)
ClientImport(ctx context.Context, ref FileRef) (cid.Cid, error)
ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
ClientListDeals(ctx context.Context) ([]DealInfo, error)
ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error)
ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error
ClientRetrieve(ctx context.Context, order RetrievalOrder, ref FileRef) error
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
// ClientUnimport removes references to the specified file from filestore
@ -155,6 +155,11 @@ type FullNode interface {
PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)
}
type FileRef struct {
Path string
IsCAR bool
}
type MinerSectors struct {
Pset uint64
Sset uint64

12
api/apistruct/struct.go
View File

@ -93,14 +93,14 @@ type FullNodeStruct struct {
WalletExport func(context.Context, address.Address) (*types.KeyInfo, error) `perm:"admin"`
WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"`
ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"admin"`
ClientImport func(ctx context.Context, ref api.FileRef) (cid.Cid, error) `perm:"admin"`
ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"`
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error) `perm:"admin"`
ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"`
ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"`
ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, path string) error `perm:"admin"`
ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error) `perm:"read"`
@ -241,8 +241,8 @@ func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, e
return c.Internal.ClientListImports(ctx)
}
func (c *FullNodeStruct) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
return c.Internal.ClientImport(ctx, path)
func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
return c.Internal.ClientImport(ctx, ref)
}
func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
@ -264,8 +264,8 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e
return c.Internal.ClientListDeals(ctx)
}
func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error {
return c.Internal.ClientRetrieve(ctx, order, path)
func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error {
return c.Internal.ClientRetrieve(ctx, order, ref)
}
func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {

6
api/test/deals.go
View File

@ -131,7 +131,11 @@ loop:
t.Fatal(err)
}
err = client.ClientRetrieve(ctx, offers[0].Order(caddr), filepath.Join(rpath, "ret"))
ref := api.FileRef{
Path: filepath.Join(rpath, "ret"),
IsCAR: false,
}
err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
if err != nil {
t.Fatalf("%+v", err)
}

22
cli/client.go
View File

@ -38,6 +38,12 @@ var clientImportCmd = &cli.Command{
Name: "import",
Usage: "Import data",
ArgsUsage: "[inputPath]",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "car",
Usage: "export to a car file instead of a regular file",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
@ -50,7 +56,11 @@ var clientImportCmd = &cli.Command{
return err
}
c, err := api.ClientImport(ctx, absPath)
ref := lapi.FileRef{
Path: absPath,
IsCAR: cctx.Bool("car"),
}
c, err := api.ClientImport(ctx, ref)
if err != nil {
return err
}
@ -251,6 +261,10 @@ var clientRetrieveCmd = &cli.Command{
Name: "address",
Usage: "address to use for transactions",
},
&cli.BoolFlag{
Name: "car",
Usage: "export to a car file instead of a regular file",
},
},
Action: func(cctx *cli.Context) error {
if cctx.NArg() != 2 {
@ -304,7 +318,11 @@ var clientRetrieveCmd = &cli.Command{
return nil
}
if err := api.ClientRetrieve(ctx, offers[0].Order(payer), cctx.Args().Get(1)); err != nil {
ref := lapi.FileRef{
Path: cctx.Args().Get(1),
IsCAR: cctx.Bool("car"),
}
if err := api.ClientRetrieve(ctx, offers[0].Order(payer), ref); err != nil {
return xerrors.Errorf("Retrieval Failed: %w", err)
}

12
lotuspond/front/src/Client.js
View File

@ -71,7 +71,12 @@ class Client extends React.Component {
let file = await this.props.pondClient.call('Pond.CreateRandomFile', [
this.state.kbs * 1000
]) // 1024 won't fit in 1k blocks :(
let cid = await this.props.client.call('Filecoin.ClientImport', [file])
let cid = await this.props.client.call('Filecoin.ClientImport', [
{
Path: file,
IsCar: false
}
])
let dealcid = await this.props.client.call('Filecoin.ClientStartDeal', [
cid,
this.state.miner,
@ -100,7 +105,10 @@ class Client extends React.Component {
await this.props.client.call('Filecoin.ClientRetrieve', [
order,
'/dev/null'
{
Path: '/dev/null',
IsCAR: false
}
])
}

37
node/impl/client/client.go
View File

@ -10,6 +10,7 @@ import (
"golang.org/x/xerrors"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-car"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore"
chunker "github.com/ipfs/go-ipfs-chunker"
@ -194,17 +195,31 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe
return out, nil
}
func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
f, err := os.Open(path)
func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
f, err := os.Open(ref.Path)
if err != nil {
return cid.Undef, err
}
if ref.IsCAR {
result, err := car.LoadCar(a.Blockstore, f)
if err != nil {
return cid.Undef, err
}
if len(result.Roots) != 1 {
return cid.Undef, xerrors.New("cannot import car with more than one root")
}
return result.Roots[0], nil
}
stat, err := f.Stat()
if err != nil {
return cid.Undef, err
}
file, err := files.NewReaderPathFile(path, f, stat)
file, err := files.NewReaderPathFile(ref.Path, f, stat)
if err != nil {
return cid.Undef, err
}
@ -288,7 +303,7 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) {
}
}
func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error {
func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error {
if order.MinerPeerID == "" {
pid, err := a.StateMinerPeerID(ctx, order.Miner, types.EmptyTSK)
if err != nil {
@ -336,6 +351,18 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
unsubscribe()
if ref.IsCAR {
f, err := os.Open(ref.Path)
if err != nil {
return err
}
err = car.WriteCar(ctx, a.LocalDAG, []cid.Cid{order.Root}, f)
if err != nil {
return err
}
return f.Close()
}
nd, err := a.LocalDAG.Get(ctx, order.Root)
if err != nil {
return xerrors.Errorf("ClientRetrieve: %w", err)
@ -344,7 +371,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
if err != nil {
return xerrors.Errorf("ClientRetrieve: %w", err)
}
return files.WriteTo(file, path)
return files.WriteTo(file, ref.Path)
}
func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {