changes as per review

This commit is contained in:
aarshkshah1992
2021-07-07 14:10:59 +05:30
parent ba236fe258
commit f8d32f5328
7 changed files with 92 additions and 28 deletions
+36 -11
View File
@@ -84,6 +84,8 @@ type API struct {
DataTransfer dtypes.ClientDataTransfer
Host host.Host
RetrievalStoreMgr dtypes.ClientRetrievalStoreManager
}
func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.ChainEpoch) abi.ChainEpoch {
@@ -619,10 +621,6 @@ func (a *API) ClientCancelRetrievalDeal(ctx context.Context, dealID retrievalmar
}
func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) error {
if ref == nil || ref.Path == "" {
return xerrors.New("must pass output file path for the retrieval deal")
}
events := make(chan marketevents.RetrievalEvent)
go a.clientRetrieve(ctx, order, ref, events)
@@ -643,10 +641,6 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
}
func (a *API) ClientRetrieveWithEvents(ctx context.Context, order api.RetrievalOrder, ref *api.FileRef) (<-chan marketevents.RetrievalEvent, error) {
if ref == nil || ref.Path == "" {
return nil, xerrors.New("must pass output file path for the retrieval deal")
}
events := make(chan marketevents.RetrievalEvent)
go a.clientRetrieve(ctx, order, ref, events)
return events, nil
@@ -783,6 +777,37 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
carV2FilePath = order.LocalCARV2FilePath
}
// TODO We only support this currently for the IPFS Retrieval use case
// where users want to write out filecoin retrievals directly to IPFS.
// If users haven' configured the Ipfs retrieval flag, the blockstore we get here will be a "no-op" blockstore.
// write out the CARv2 file to the retrieval block-store (which is really an IPFS node behind the scenes).
rs, err := a.RetrievalStoreMgr.NewStore()
defer a.RetrievalStoreMgr.ReleaseStore(rs) //nolint:errcheck
if err != nil {
finish(xerrors.Errorf("Error setting up new store: %w", err))
return
}
if rs.IsIPFSRetrieval() {
// write out the CARv1 blocks of the CARv2 file to the IPFS blockstore.
carv2Reader, err := carv2.NewReaderMmap(carV2FilePath)
if err != nil {
finish(err)
return
}
defer carv2Reader.Close() //nolint:errcheck
if _, err := car.LoadCar(rs.Blockstore(), carv2Reader.CarV1Reader()); err != nil {
finish(err)
return
}
}
// If ref is nil, it only fetches the data into the configured blockstore.
if ref == nil {
finish(nil)
return
}
if ref.IsCAR {
// user wants a CAR file, transform the CARv2 to a CARv1 and write it out.
f, err := os.OpenFile(ref.Path, os.O_CREATE|os.O_WRONLY, 0644)
@@ -806,13 +831,13 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
rw, err := blockstore.OpenReadOnly(carV2FilePath)
readOnly, err := blockstore.OpenReadOnly(carV2FilePath)
if err != nil {
finish(err)
return
}
defer rw.Close() //nolint:errcheck
bsvc := blockservice.New(rw, offline.Exchange(rw))
defer readOnly.Close() //nolint:errcheck
bsvc := blockservice.New(readOnly, offline.Exchange(readOnly))
dag := merkledag.NewDAGService(bsvc)
nd, err := dag.Get(ctx, order.Root)
+26 -1
View File
@@ -73,7 +73,7 @@ func TestImportNormalFileToCARv2(t *testing.T) {
a := &API{
Imports: &importmgr.Mgr{},
}
importID := rand.Uint64()
importID := importmgr.ImportID(rand.Uint64())
inputFilePath, inputContents := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
@@ -148,6 +148,31 @@ func TestTransformCarv1ToCARv2(t *testing.T) {
require.Equal(t, bzin, bzout)
}
func TestLoadCARv2ToBlockstore(t *testing.T) {
inputFilePath, _ := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
carv1FilePath := genCARv1(t, inputFilePath)
defer os.Remove(carv1FilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
bs := bstore.NewMemorySync()
carv2, err := carv2.NewReaderMmap(outputCARv2)
require.NoError(t, err)
defer carv2.Close() //nolint:errcheck
header, err := car.LoadCar(bs, carv2.CarV1Reader())
require.NoError(t, err)
require.EqualValues(t, root, header.Roots[0])
require.EqualValues(t, 1, header.Version)
}
func genCARv1(t *testing.T, normalFilePath string) string {
ctx := context.Background()
bs := bstore.NewMemorySync()