cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

tests for storage client import impls

Browse Source
This commit is contained in:
aarshkshah1992
2021-07-06 12:40:57 +05:30
parent a07a0935fe
commit e041fe130b
4 changed files with 202 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/client/client.go
+4 -4
View File
@@ -808,7 +808,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
return
}
rw, err := blockstore.OpenReadOnly(carV2FilePath, false)
rw, err := blockstore.OpenReadOnly(carV2FilePath)
if err != nil {
finish(err)
return
@@ -984,7 +984,7 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
return api.DataSize{}, xerrors.New("no CARv2 file for root")
}
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
if err != nil {
return api.DataSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
}
@@ -1015,7 +1015,7 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
}
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
if err != nil {
return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
}
@@ -1053,7 +1053,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
// generate a deterministic CARv1 payload from the UnixFS DAG by doing an IPLD
// traversal over the Unixfs DAG in the CARv2 file using the "all selector" i.e the entire DAG selector.
rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File, true)
rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File)
if err != nil {
return xerrors.Errorf("failed to open read only CARv2 blockstore: %w", err)
}
node/impl/client/import_test.go
+190
View File
@@ -0,0 +1,190 @@
package client
import (
"bufio"
"context"
"io"
"io/ioutil"
"math/rand"
"os"
"strings"
"testing"
bstore "github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/node/repo/importmgr"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
offline "github.com/ipfs/go-ipfs-exchange-offline"
files "github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-merkledag"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/ipld/go-car"
carv2 "github.com/ipld/go-car/v2"
"github.com/ipld/go-car/v2/blockstore"
basicnode "github.com/ipld/go-ipld-prime/node/basic"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
"github.com/stretchr/testify/require"
)
func TestImportNormalFileToUnixfsDAG(t *testing.T) {
ctx := context.Background()
inputPath, inputContents := genNormalInputFile(t)
defer os.Remove(inputPath) //nolint:errcheck
carv2File := genTmpFile(t)
defer os.Remove(carv2File) //nolint:errcheck
// import a normal file to a Unixfs DAG using a CARv2 read-write blockstore and flush it out to a CARv2 file.
tempCARv2Store, err := blockstore.NewReadWrite(carv2File, []cid.Cid{})
require.NoError(t, err)
bsvc := blockservice.New(tempCARv2Store, offline.Exchange(tempCARv2Store))
root, err := importNormalFileToUnixfsDAG(ctx, inputPath, merkledag.NewDAGService(bsvc))
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
require.NoError(t, tempCARv2Store.Finalize())
// convert the CARv2 file to a normal file again and ensure the contents match.
readOnly, err := blockstore.OpenReadOnly(carv2File)
require.NoError(t, err)
defer readOnly.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
nd, err := dag.Get(ctx, root)
require.NoError(t, err)
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
require.NoError(t, err)
tmpOutput := genTmpFile(t)
defer os.Remove(tmpOutput) //nolint:errcheck
require.NoError(t, files.WriteTo(file, tmpOutput))
// ensure contents of the initial input file and the output file are identical.
fo, err := os.Open(tmpOutput)
require.NoError(t, err)
bz2, err := ioutil.ReadAll(fo)
require.NoError(t, err)
require.NoError(t, fo.Close())
require.Equal(t, inputContents, bz2)
}
func TestImportNormalFileToCARv2(t *testing.T) {
ctx := context.Background()
a := &API{
Imports: &importmgr.Mgr{},
}
importID := rand.Uint64()
inputFilePath, inputContents := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
root, err := a.importNormalFileToCARv2(ctx, importID, inputFilePath, outputCARv2)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
// convert the CARv2 to a normal file again and ensure the contents match
readOnly, err := blockstore.OpenReadOnly(outputCARv2)
require.NoError(t, err)
defer readOnly.Close() //nolint:errcheck
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
nd, err := dag.Get(ctx, root)
require.NoError(t, err)
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
require.NoError(t, err)
tmpOutput := genTmpFile(t)
defer os.Remove(tmpOutput) //nolint:errcheck
require.NoError(t, files.WriteTo(file, tmpOutput))
// ensure contents of the initial input file and the output file are identical.
fo, err := os.Open(tmpOutput)
require.NoError(t, err)
bz2, err := ioutil.ReadAll(fo)
require.NoError(t, err)
require.NoError(t, fo.Close())
require.Equal(t, inputContents, bz2)
}
func TestTransformCarv1ToCARv2(t *testing.T) {
inputFilePath, _ := genNormalInputFile(t)
defer os.Remove(inputFilePath) //nolint:errcheck
carv1FilePath := genCARv1(t, inputFilePath)
defer os.Remove(carv1FilePath) //nolint:errcheck
outputCARv2 := genTmpFile(t)
defer os.Remove(outputCARv2) //nolint:errcheck
root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
require.NoError(t, err)
require.NotEqual(t, cid.Undef, root)
// assert what we got back is a valid CARv2 and that the CARv1 payload is exactly what we gave it
f2, err := os.Open(outputCARv2)
require.NoError(t, err)
hd, _, err := car.ReadHeader(bufio.NewReader(f2))
require.NoError(t, err)
require.EqualValues(t, 2, hd.Version)
require.NoError(t, f2.Close())
v2r, err := carv2.NewReaderMmap(outputCARv2)
require.NoError(t, err)
bzout, err := ioutil.ReadAll(v2r.CarV1Reader())
require.NoError(t, err)
require.NotNil(t, bzout)
require.NoError(t, v2r.Close())
fi, err := os.Open(carv1FilePath)
require.NoError(t, err)
bzin, err := ioutil.ReadAll(fi)
require.NoError(t, err)
require.NoError(t, fi.Close())
require.NotNil(t, bzin)
require.Equal(t, bzin, bzout)
}
func genCARv1(t *testing.T, normalFilePath string) string {
ctx := context.Background()
bs := bstore.NewMemorySync()
root, err := importNormalFileToUnixfsDAG(ctx, normalFilePath, merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))))
require.NoError(t, err)
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: root, Selector: allSelector}})
f, err := os.CreateTemp("", "")
require.NoError(t, err)
require.NoError(t, sc.Write(f))
_, err = f.Seek(0, io.SeekStart)
require.NoError(t, err)
hd, _, err := car.ReadHeader(bufio.NewReader(f))
require.NoError(t, err)
require.EqualValues(t, 1, hd.Version)
require.NoError(t, f.Close())
return f.Name()
}
func genTmpFile(t *testing.T) string {
f, err := os.CreateTemp("", "")
require.NoError(t, err)
require.NoError(t, f.Close())
return f.Name()
}
func genNormalInputFile(t *testing.T) (filepath string, contents []byte) {
s := strings.Repeat("abcde", 100)
tmp, err := os.CreateTemp("", "")
require.NoError(t, err)
_, err = io.Copy(tmp, strings.NewReader(s))
require.NoError(t, err)
require.NoError(t, tmp.Close())
return tmp.Name(), []byte(s)
}