tests for storage client import impls

go.mod

@@ -66,7 +66,7 @@ require (
 	github.com/ipfs/go-bitswap v0.3.2
 	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-blockservice v0.1.4
-	github.com/ipfs/go-cid v0.0.7
+	github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672
 	github.com/ipfs/go-cidutil v0.0.2
 	github.com/ipfs/go-datastore v0.4.5
 	github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e
@@ -95,7 +95,7 @@ require (
 	github.com/ipfs/go-unixfs v0.2.6
 	github.com/ipfs/interface-go-ipfs-core v0.2.3
 	github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
-	github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a
+	github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1
 	github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/lib/pq v1.7.0

go.sum

@@ -588,8 +588,9 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
 github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
 github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
-github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
 github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
+github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672 h1:PabVicIEIt7qUwx5gu80wZsALHUZ4Zux37M+x0n/Erk=
+github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o=
 github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo=
 github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s=
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@@ -752,8 +753,8 @@ github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D
 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
 github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
 github.com/ipld/go-car/v2 v2.0.0-20210629123041-a9ebfcacd98c/go.mod h1:/hLZQELe+MHdeBcE+CBonAR90e6rrRVexIfyBlRplDY=
-github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a h1:m5/3djh93XjEH/p+WQjd3T1Fle1cqCYpfKEiaxoLrwU=
+github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1 h1:QIUM+raVYcnjqaK/KxLKtZZ7/FrJMtC8vzuo0mCKaFY=
-github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a/go.mod h1:/hLZQELe+MHdeBcE+CBonAR90e6rrRVexIfyBlRplDY=
+github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
 github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
 github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
 github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
@@ -1717,8 +1718,9 @@ golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
+golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

node/impl/client/client.go

@@ -808,7 +808,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
 		return
 	}
 
-	rw, err := blockstore.OpenReadOnly(carV2FilePath, false)
+	rw, err := blockstore.OpenReadOnly(carV2FilePath)
 	if err != nil {
 		finish(err)
 		return
@@ -984,7 +984,7 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
 		return api.DataSize{}, xerrors.New("no CARv2 file for root")
 	}
 
-	rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
+	rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
 	if err != nil {
 		return api.DataSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
 	}
@@ -1015,7 +1015,7 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
 		return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
 	}
 
-	rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
+	rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
 	if err != nil {
 		return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
 	}
@@ -1053,7 +1053,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
 
 	// generate a deterministic CARv1 payload from the UnixFS DAG by doing an IPLD
 	// traversal over the Unixfs DAG in the CARv2 file using the "all selector" i.e the entire DAG selector.
-	rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File, true)
+	rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File)
 	if err != nil {
 		return xerrors.Errorf("failed to open read only CARv2 blockstore: %w", err)
 	}

node/impl/client/import_test.go

@@ -0,0 +1,190 @@
+package client
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+
+	bstore "github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/node/repo/importmgr"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	files "github.com/ipfs/go-ipfs-files"
+	"github.com/ipfs/go-merkledag"
+	unixfile "github.com/ipfs/go-unixfs/file"
+	"github.com/ipld/go-car"
+	carv2 "github.com/ipld/go-car/v2"
+	"github.com/ipld/go-car/v2/blockstore"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/traversal/selector"
+	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImportNormalFileToUnixfsDAG(t *testing.T) {
+	ctx := context.Background()
+	inputPath, inputContents := genNormalInputFile(t)
+	defer os.Remove(inputPath) //nolint:errcheck
+	carv2File := genTmpFile(t)
+	defer os.Remove(carv2File) //nolint:errcheck
+
+	// import a normal file to a Unixfs DAG using a CARv2 read-write blockstore and flush it out to a CARv2 file.
+	tempCARv2Store, err := blockstore.NewReadWrite(carv2File, []cid.Cid{})
+	require.NoError(t, err)
+	bsvc := blockservice.New(tempCARv2Store, offline.Exchange(tempCARv2Store))
+	root, err := importNormalFileToUnixfsDAG(ctx, inputPath, merkledag.NewDAGService(bsvc))
+	require.NoError(t, err)
+	require.NotEqual(t, cid.Undef, root)
+	require.NoError(t, tempCARv2Store.Finalize())
+
+	// convert the CARv2 file to a normal file again and ensure the contents match.
+	readOnly, err := blockstore.OpenReadOnly(carv2File)
+	require.NoError(t, err)
+	defer readOnly.Close() //nolint:errcheck
+	dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
+
+	nd, err := dag.Get(ctx, root)
+	require.NoError(t, err)
+	file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
+	require.NoError(t, err)
+
+	tmpOutput := genTmpFile(t)
+	defer os.Remove(tmpOutput) //nolint:errcheck
+	require.NoError(t, files.WriteTo(file, tmpOutput))
+
+	// ensure contents of the initial input file and the output file are identical.
+
+	fo, err := os.Open(tmpOutput)
+	require.NoError(t, err)
+	bz2, err := ioutil.ReadAll(fo)
+	require.NoError(t, err)
+	require.NoError(t, fo.Close())
+	require.Equal(t, inputContents, bz2)
+}
+
+func TestImportNormalFileToCARv2(t *testing.T) {
+	ctx := context.Background()
+	a := &API{
+		Imports: &importmgr.Mgr{},
+	}
+	importID := rand.Uint64()
+
+	inputFilePath, inputContents := genNormalInputFile(t)
+	defer os.Remove(inputFilePath) //nolint:errcheck
+
+	outputCARv2 := genTmpFile(t)
+	defer os.Remove(outputCARv2) //nolint:errcheck
+
+	root, err := a.importNormalFileToCARv2(ctx, importID, inputFilePath, outputCARv2)
+	require.NoError(t, err)
+	require.NotEqual(t, cid.Undef, root)
+
+	// convert the CARv2 to a normal file again and ensure the contents match
+	readOnly, err := blockstore.OpenReadOnly(outputCARv2)
+	require.NoError(t, err)
+	defer readOnly.Close() //nolint:errcheck
+	dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
+
+	nd, err := dag.Get(ctx, root)
+	require.NoError(t, err)
+	file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
+	require.NoError(t, err)
+
+	tmpOutput := genTmpFile(t)
+	defer os.Remove(tmpOutput) //nolint:errcheck
+	require.NoError(t, files.WriteTo(file, tmpOutput))
+
+	// ensure contents of the initial input file and the output file are identical.
+	fo, err := os.Open(tmpOutput)
+	require.NoError(t, err)
+	bz2, err := ioutil.ReadAll(fo)
+	require.NoError(t, err)
+	require.NoError(t, fo.Close())
+	require.Equal(t, inputContents, bz2)
+}
+
+func TestTransformCarv1ToCARv2(t *testing.T) {
+	inputFilePath, _ := genNormalInputFile(t)
+	defer os.Remove(inputFilePath) //nolint:errcheck
+
+	carv1FilePath := genCARv1(t, inputFilePath)
+	defer os.Remove(carv1FilePath) //nolint:errcheck
+
+	outputCARv2 := genTmpFile(t)
+	defer os.Remove(outputCARv2) //nolint:errcheck
+
+	root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
+	require.NoError(t, err)
+	require.NotEqual(t, cid.Undef, root)
+
+	// assert what we got back is a valid CARv2 and that the CARv1 payload is exactly what we gave it
+	f2, err := os.Open(outputCARv2)
+	require.NoError(t, err)
+	hd, _, err := car.ReadHeader(bufio.NewReader(f2))
+	require.NoError(t, err)
+	require.EqualValues(t, 2, hd.Version)
+	require.NoError(t, f2.Close())
+
+	v2r, err := carv2.NewReaderMmap(outputCARv2)
+	require.NoError(t, err)
+	bzout, err := ioutil.ReadAll(v2r.CarV1Reader())
+	require.NoError(t, err)
+	require.NotNil(t, bzout)
+	require.NoError(t, v2r.Close())
+
+	fi, err := os.Open(carv1FilePath)
+	require.NoError(t, err)
+	bzin, err := ioutil.ReadAll(fi)
+	require.NoError(t, err)
+	require.NoError(t, fi.Close())
+	require.NotNil(t, bzin)
+
+	require.Equal(t, bzin, bzout)
+}
+
+func genCARv1(t *testing.T, normalFilePath string) string {
+	ctx := context.Background()
+	bs := bstore.NewMemorySync()
+	root, err := importNormalFileToUnixfsDAG(ctx, normalFilePath, merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))))
+	require.NoError(t, err)
+
+	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
+	allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
+		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
+	sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: root, Selector: allSelector}})
+	f, err := os.CreateTemp("", "")
+	require.NoError(t, err)
+	require.NoError(t, sc.Write(f))
+
+	_, err = f.Seek(0, io.SeekStart)
+	require.NoError(t, err)
+	hd, _, err := car.ReadHeader(bufio.NewReader(f))
+	require.NoError(t, err)
+	require.EqualValues(t, 1, hd.Version)
+
+	require.NoError(t, f.Close())
+	return f.Name()
+}
+
+func genTmpFile(t *testing.T) string {
+	f, err := os.CreateTemp("", "")
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	return f.Name()
+}
+
+func genNormalInputFile(t *testing.T) (filepath string, contents []byte) {
+	s := strings.Repeat("abcde", 100)
+	tmp, err := os.CreateTemp("", "")
+	require.NoError(t, err)
+	_, err = io.Copy(tmp, strings.NewReader(s))
+	require.NoError(t, err)
+	require.NoError(t, tmp.Close())
+	return tmp.Name(), []byte(s)
+}