feat(client): add car import/export option

adds option on client file import and client retrieval to read/write to CAR file

api/api_full.go

@@ -89,13 +89,13 @@ type FullNode interface {
 	// Other
 
 	// ClientImport imports file under the specified path into filestore
-	ClientImport(ctx context.Context, path string) (cid.Cid, error)
+	ClientImport(ctx context.Context, ref FileRef) (cid.Cid, error)
 	ClientStartDeal(ctx context.Context, params *StartDealParams) (*cid.Cid, error)
 	ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
 	ClientListDeals(ctx context.Context) ([]DealInfo, error)
 	ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
 	ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error)
-	ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error
+	ClientRetrieve(ctx context.Context, order RetrievalOrder, ref FileRef) error
 	ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error)
 
 	// ClientUnimport removes references to the specified file from filestore
@@ -155,6 +155,11 @@ type FullNode interface {
 	PaychVoucherSubmit(context.Context, address.Address, *paych.SignedVoucher) (cid.Cid, error)
 }
 
+type FileRef struct {
+	Path  string
+	IsCAR bool
+}
+
 type MinerSectors struct {
 	Pset uint64
 	Sset uint64

api/apistruct/struct.go

@@ -93,14 +93,14 @@ type FullNodeStruct struct {
 		WalletExport         func(context.Context, address.Address) (*types.KeyInfo, error)                       `perm:"admin"`
 		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
 
-		ClientImport      func(ctx context.Context, path string) (cid.Cid, error)                                              `perm:"admin"`
+		ClientImport      func(ctx context.Context, ref api.FileRef) (cid.Cid, error)                                              `perm:"admin"`
 		ClientListImports func(ctx context.Context) ([]api.Import, error)                                                      `perm:"write"`
 		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                `perm:"write"`
 		ClientFindData    func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error)                                    `perm:"read"`
 		ClientStartDeal   func(ctx context.Context, params *api.StartDealParams) (*cid.Cid, error)                             `perm:"admin"`
 		ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error)                                                `perm:"read"`
 		ClientListDeals   func(ctx context.Context) ([]api.DealInfo, error)                                                    `perm:"write"`
-		ClientRetrieve    func(ctx context.Context, order api.RetrievalOrder, path string) error                               `perm:"admin"`
+		ClientRetrieve    func(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error                               `perm:"admin"`
 		ClientQueryAsk    func(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) `perm:"read"`
 
 		StateMinerSectors       func(context.Context, address.Address, types.TipSetKey) ([]*api.ChainSectorInfo, error)                      `perm:"read"`
@@ -241,8 +241,8 @@ func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, e
 	return c.Internal.ClientListImports(ctx)
 }
 
-func (c *FullNodeStruct) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
+func (c *FullNodeStruct) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
-	return c.Internal.ClientImport(ctx, path)
+	return c.Internal.ClientImport(ctx, ref)
 }
 
 func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
@@ -264,8 +264,8 @@ func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, e
 	return c.Internal.ClientListDeals(ctx)
 }
 
-func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error {
+func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error {
-	return c.Internal.ClientRetrieve(ctx, order, path)
+	return c.Internal.ClientRetrieve(ctx, order, ref)
 }
 
 func (c *FullNodeStruct) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {

api/test/deals.go

@@ -131,7 +131,11 @@ loop:
 		t.Fatal(err)
 	}
 
-	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), filepath.Join(rpath, "ret"))
+	ref := api.FileRef{
+		Path:  filepath.Join(rpath, "ret"),
+		IsCAR: false,
+	}
+	err = client.ClientRetrieve(ctx, offers[0].Order(caddr), ref)
 	if err != nil {
 		t.Fatalf("%+v", err)
 	}

cli/client.go

@@ -38,6 +38,12 @@ var clientImportCmd = &cli.Command{
 	Name:      "import",
 	Usage:     "Import data",
 	ArgsUsage: "[inputPath]",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "car",
+			Usage: "export to a car file instead of a regular file",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
@@ -50,7 +56,11 @@ var clientImportCmd = &cli.Command{
 			return err
 		}
 
-		c, err := api.ClientImport(ctx, absPath)
+		ref := lapi.FileRef{
+			Path:  absPath,
+			IsCAR: cctx.Bool("car"),
+		}
+		c, err := api.ClientImport(ctx, ref)
 		if err != nil {
 			return err
 		}
@@ -251,6 +261,10 @@ var clientRetrieveCmd = &cli.Command{
 			Name:  "address",
 			Usage: "address to use for transactions",
 		},
+		&cli.BoolFlag{
+			Name:  "car",
+			Usage: "export to a car file instead of a regular file",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.NArg() != 2 {
@@ -304,7 +318,11 @@ var clientRetrieveCmd = &cli.Command{
 			return nil
 		}
 
-		if err := api.ClientRetrieve(ctx, offers[0].Order(payer), cctx.Args().Get(1)); err != nil {
+		ref := lapi.FileRef{
+			Path:  cctx.Args().Get(1),
+			IsCAR: cctx.Bool("car"),
+		}
+		if err := api.ClientRetrieve(ctx, offers[0].Order(payer), ref); err != nil {
 			return xerrors.Errorf("Retrieval Failed: %w", err)
 		}
 

lotuspond/front/src/Client.js

@@ -71,7 +71,12 @@ class Client extends React.Component {
     let file = await this.props.pondClient.call('Pond.CreateRandomFile', [
       this.state.kbs * 1000
     ]) // 1024 won't fit in 1k blocks :(
-    let cid = await this.props.client.call('Filecoin.ClientImport', [file])
+    let cid = await this.props.client.call('Filecoin.ClientImport', [
+      {
+        Path: file,
+        IsCar: false
+      }
+    ])
     let dealcid = await this.props.client.call('Filecoin.ClientStartDeal', [
       cid,
       this.state.miner,
@@ -100,7 +105,10 @@ class Client extends React.Component {
 
     await this.props.client.call('Filecoin.ClientRetrieve', [
       order,
-      '/dev/null'
+      {
+        Path: '/dev/null',
+        IsCAR: false
+      }
     ])
   }
 

node/impl/client/client.go

@@ -10,6 +10,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-car"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-filestore"
 	chunker "github.com/ipfs/go-ipfs-chunker"
@@ -194,17 +195,31 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe
 	return out, nil
 }
 
-func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
+func (a *API) ClientImport(ctx context.Context, ref api.FileRef) (cid.Cid, error) {
-	f, err := os.Open(path)
+	f, err := os.Open(ref.Path)
 	if err != nil {
 		return cid.Undef, err
 	}
+
+	if ref.IsCAR {
+		result, err := car.LoadCar(a.Blockstore, f)
+		if err != nil {
+			return cid.Undef, err
+		}
+
+		if len(result.Roots) != 1 {
+			return cid.Undef, xerrors.New("cannot import car with more than one root")
+		}
+
+		return result.Roots[0], nil
+	}
+
 	stat, err := f.Stat()
 	if err != nil {
 		return cid.Undef, err
 	}
 
-	file, err := files.NewReaderPathFile(path, f, stat)
+	file, err := files.NewReaderPathFile(ref.Path, f, stat)
 	if err != nil {
 		return cid.Undef, err
 	}
@@ -288,7 +303,7 @@ func (a *API) ClientListImports(ctx context.Context) ([]api.Import, error) {
 	}
 }
 
-func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error {
+func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref api.FileRef) error {
 	if order.MinerPeerID == "" {
 		pid, err := a.StateMinerPeerID(ctx, order.Miner, types.EmptyTSK)
 		if err != nil {
@@ -336,6 +351,18 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
 
 	unsubscribe()
 
+	if ref.IsCAR {
+		f, err := os.Open(ref.Path)
+		if err != nil {
+			return err
+		}
+		err = car.WriteCar(ctx, a.LocalDAG, []cid.Cid{order.Root}, f)
+		if err != nil {
+			return err
+		}
+		return f.Close()
+	}
+
 	nd, err := a.LocalDAG.Get(ctx, order.Root)
 	if err != nil {
 		return xerrors.Errorf("ClientRetrieve: %w", err)
@@ -344,7 +371,7 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
 	if err != nil {
 		return xerrors.Errorf("ClientRetrieve: %w", err)
 	}
-	return files.WriteTo(file, path)
+	return files.WriteTo(file, ref.Path)
 }
 
 func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*storagemarket.SignedStorageAsk, error) {