tests for storage client import impls
This commit is contained in:
aarshkshah1992
parent
a07a0935fe
commit
e041fe130b
4
go.mod
4
go.mod
@ -66,7 +66,7 @@ require (
|
||||
github.com/ipfs/go-bitswap v0.3.2
|
||||
github.com/ipfs/go-block-format v0.0.3
|
||||
github.com/ipfs/go-blockservice v0.1.4
|
||||
github.com/ipfs/go-cid v0.0.7
|
||||
github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672
|
||||
github.com/ipfs/go-cidutil v0.0.2
|
||||
github.com/ipfs/go-datastore v0.4.5
|
||||
github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e
|
||||
@ -95,7 +95,7 @@ require (
|
||||
github.com/ipfs/go-unixfs v0.2.6
|
||||
github.com/ipfs/interface-go-ipfs-core v0.2.3
|
||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1
|
||||
github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
|
||||
github.com/kelseyhightower/envconfig v1.4.0
|
||||
github.com/lib/pq v1.7.0
|
||||
|
||||
10
go.sum
10
go.sum
@ -588,8 +588,9 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj
|
||||
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
|
||||
github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
|
||||
github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672 h1:PabVicIEIt7qUwx5gu80wZsALHUZ4Zux37M+x0n/Erk=
|
||||
github.com/ipfs/go-cid v0.0.8-0.20210702173502-41f2377d9672/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o=
|
||||
github.com/ipfs/go-cidutil v0.0.2 h1:CNOboQf1t7Qp0nuNh8QMmhJs0+Q//bRL1axtCnIB1Yo=
|
||||
github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s=
|
||||
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
|
||||
@ -752,8 +753,8 @@ github.com/ipld/go-car v0.1.1-0.20200923150018-8cdef32e2da4/go.mod h1:xrMEcuSq+D
|
||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d h1:iphSzTuPqyDgH7WUVZsdqUnQNzYgIblsVr1zhVNA33U=
|
||||
github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d/go.mod h1:2Gys8L8MJ6zkh1gktTSXreY63t4UbyvNp5JaudTyxHQ=
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210629123041-a9ebfcacd98c/go.mod h1:/hLZQELe+MHdeBcE+CBonAR90e6rrRVexIfyBlRplDY=
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a h1:m5/3djh93XjEH/p+WQjd3T1Fle1cqCYpfKEiaxoLrwU=
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210630160528-7bfe32ea582a/go.mod h1:/hLZQELe+MHdeBcE+CBonAR90e6rrRVexIfyBlRplDY=
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1 h1:QIUM+raVYcnjqaK/KxLKtZZ7/FrJMtC8vzuo0mCKaFY=
|
||||
github.com/ipld/go-car/v2 v2.0.0-20210705172946-d1ec44b36fa1/go.mod h1:Ueq4zx/SNx7yHwmfr9xKlKpXxRCMM6wyqC8B0rv9oig=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20200428162820-8b59dc292b8e/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8=
|
||||
github.com/ipld/go-ipld-prime v0.5.1-0.20200828233916-988837377a7f/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
|
||||
@ -1717,8 +1718,9 @@ golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
|
||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
|
||||
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
|
||||
@ -808,7 +808,7 @@ func (a *API) clientRetrieve(ctx context.Context, order api.RetrievalOrder, ref
|
||||
return
|
||||
}
|
||||
|
||||
rw, err := blockstore.OpenReadOnly(carV2FilePath, false)
|
||||
rw, err := blockstore.OpenReadOnly(carV2FilePath)
|
||||
if err != nil {
|
||||
finish(err)
|
||||
return
|
||||
@ -984,7 +984,7 @@ func (a *API) ClientDealSize(ctx context.Context, root cid.Cid) (api.DataSize, e
|
||||
return api.DataSize{}, xerrors.New("no CARv2 file for root")
|
||||
}
|
||||
|
||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
|
||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
|
||||
if err != nil {
|
||||
return api.DataSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
|
||||
}
|
||||
@ -1015,7 +1015,7 @@ func (a *API) ClientDealPieceCID(ctx context.Context, root cid.Cid) (api.DataCID
|
||||
return api.DataCIDSize{}, xerrors.New("no CARv2 file for root")
|
||||
}
|
||||
|
||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath, false)
|
||||
rdOnly, err := blockstore.OpenReadOnly(carv2FilePath)
|
||||
if err != nil {
|
||||
return api.DataCIDSize{}, xerrors.Errorf("failed to open read only blockstore: %w", err)
|
||||
}
|
||||
@ -1053,7 +1053,7 @@ func (a *API) ClientGenCar(ctx context.Context, ref api.FileRef, outputPath stri
|
||||
|
||||
// generate a deterministic CARv1 payload from the UnixFS DAG by doing an IPLD
|
||||
// traversal over the Unixfs DAG in the CARv2 file using the "all selector" i.e the entire DAG selector.
|
||||
rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File, true)
|
||||
rdOnly, err := blockstore.OpenReadOnly(tmpCARv2File)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to open read only CARv2 blockstore: %w", err)
|
||||
}
|
||||
|
||||
190
node/impl/client/import_test.go
Normal file
190
node/impl/client/import_test.go
Normal file
@ -0,0 +1,190 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
bstore "github.com/filecoin-project/lotus/blockstore"
|
||||
"github.com/filecoin-project/lotus/node/repo/importmgr"
|
||||
"github.com/ipfs/go-blockservice"
|
||||
"github.com/ipfs/go-cid"
|
||||
offline "github.com/ipfs/go-ipfs-exchange-offline"
|
||||
files "github.com/ipfs/go-ipfs-files"
|
||||
"github.com/ipfs/go-merkledag"
|
||||
unixfile "github.com/ipfs/go-unixfs/file"
|
||||
"github.com/ipld/go-car"
|
||||
carv2 "github.com/ipld/go-car/v2"
|
||||
"github.com/ipld/go-car/v2/blockstore"
|
||||
basicnode "github.com/ipld/go-ipld-prime/node/basic"
|
||||
"github.com/ipld/go-ipld-prime/traversal/selector"
|
||||
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestImportNormalFileToUnixfsDAG(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
inputPath, inputContents := genNormalInputFile(t)
|
||||
defer os.Remove(inputPath) //nolint:errcheck
|
||||
carv2File := genTmpFile(t)
|
||||
defer os.Remove(carv2File) //nolint:errcheck
|
||||
|
||||
// import a normal file to a Unixfs DAG using a CARv2 read-write blockstore and flush it out to a CARv2 file.
|
||||
tempCARv2Store, err := blockstore.NewReadWrite(carv2File, []cid.Cid{})
|
||||
require.NoError(t, err)
|
||||
bsvc := blockservice.New(tempCARv2Store, offline.Exchange(tempCARv2Store))
|
||||
root, err := importNormalFileToUnixfsDAG(ctx, inputPath, merkledag.NewDAGService(bsvc))
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, cid.Undef, root)
|
||||
require.NoError(t, tempCARv2Store.Finalize())
|
||||
|
||||
// convert the CARv2 file to a normal file again and ensure the contents match.
|
||||
readOnly, err := blockstore.OpenReadOnly(carv2File)
|
||||
require.NoError(t, err)
|
||||
defer readOnly.Close() //nolint:errcheck
|
||||
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
|
||||
|
||||
nd, err := dag.Get(ctx, root)
|
||||
require.NoError(t, err)
|
||||
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
|
||||
require.NoError(t, err)
|
||||
|
||||
tmpOutput := genTmpFile(t)
|
||||
defer os.Remove(tmpOutput) //nolint:errcheck
|
||||
require.NoError(t, files.WriteTo(file, tmpOutput))
|
||||
|
||||
// ensure contents of the initial input file and the output file are identical.
|
||||
|
||||
fo, err := os.Open(tmpOutput)
|
||||
require.NoError(t, err)
|
||||
bz2, err := ioutil.ReadAll(fo)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fo.Close())
|
||||
require.Equal(t, inputContents, bz2)
|
||||
}
|
||||
|
||||
func TestImportNormalFileToCARv2(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
a := &API{
|
||||
Imports: &importmgr.Mgr{},
|
||||
}
|
||||
importID := rand.Uint64()
|
||||
|
||||
inputFilePath, inputContents := genNormalInputFile(t)
|
||||
defer os.Remove(inputFilePath) //nolint:errcheck
|
||||
|
||||
outputCARv2 := genTmpFile(t)
|
||||
defer os.Remove(outputCARv2) //nolint:errcheck
|
||||
|
||||
root, err := a.importNormalFileToCARv2(ctx, importID, inputFilePath, outputCARv2)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, cid.Undef, root)
|
||||
|
||||
// convert the CARv2 to a normal file again and ensure the contents match
|
||||
readOnly, err := blockstore.OpenReadOnly(outputCARv2)
|
||||
require.NoError(t, err)
|
||||
defer readOnly.Close() //nolint:errcheck
|
||||
dag := merkledag.NewDAGService(blockservice.New(readOnly, offline.Exchange(readOnly)))
|
||||
|
||||
nd, err := dag.Get(ctx, root)
|
||||
require.NoError(t, err)
|
||||
file, err := unixfile.NewUnixfsFile(ctx, dag, nd)
|
||||
require.NoError(t, err)
|
||||
|
||||
tmpOutput := genTmpFile(t)
|
||||
defer os.Remove(tmpOutput) //nolint:errcheck
|
||||
require.NoError(t, files.WriteTo(file, tmpOutput))
|
||||
|
||||
// ensure contents of the initial input file and the output file are identical.
|
||||
fo, err := os.Open(tmpOutput)
|
||||
require.NoError(t, err)
|
||||
bz2, err := ioutil.ReadAll(fo)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fo.Close())
|
||||
require.Equal(t, inputContents, bz2)
|
||||
}
|
||||
|
||||
func TestTransformCarv1ToCARv2(t *testing.T) {
|
||||
inputFilePath, _ := genNormalInputFile(t)
|
||||
defer os.Remove(inputFilePath) //nolint:errcheck
|
||||
|
||||
carv1FilePath := genCARv1(t, inputFilePath)
|
||||
defer os.Remove(carv1FilePath) //nolint:errcheck
|
||||
|
||||
outputCARv2 := genTmpFile(t)
|
||||
defer os.Remove(outputCARv2) //nolint:errcheck
|
||||
|
||||
root, err := transformCarToCARv2(carv1FilePath, outputCARv2)
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, cid.Undef, root)
|
||||
|
||||
// assert what we got back is a valid CARv2 and that the CARv1 payload is exactly what we gave it
|
||||
f2, err := os.Open(outputCARv2)
|
||||
require.NoError(t, err)
|
||||
hd, _, err := car.ReadHeader(bufio.NewReader(f2))
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 2, hd.Version)
|
||||
require.NoError(t, f2.Close())
|
||||
|
||||
v2r, err := carv2.NewReaderMmap(outputCARv2)
|
||||
require.NoError(t, err)
|
||||
bzout, err := ioutil.ReadAll(v2r.CarV1Reader())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, bzout)
|
||||
require.NoError(t, v2r.Close())
|
||||
|
||||
fi, err := os.Open(carv1FilePath)
|
||||
require.NoError(t, err)
|
||||
bzin, err := ioutil.ReadAll(fi)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fi.Close())
|
||||
require.NotNil(t, bzin)
|
||||
|
||||
require.Equal(t, bzin, bzout)
|
||||
}
|
||||
|
||||
func genCARv1(t *testing.T, normalFilePath string) string {
|
||||
ctx := context.Background()
|
||||
bs := bstore.NewMemorySync()
|
||||
root, err := importNormalFileToUnixfsDAG(ctx, normalFilePath, merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))))
|
||||
require.NoError(t, err)
|
||||
|
||||
ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
|
||||
allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
|
||||
ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
|
||||
sc := car.NewSelectiveCar(ctx, bs, []car.Dag{{Root: root, Selector: allSelector}})
|
||||
f, err := os.CreateTemp("", "")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sc.Write(f))
|
||||
|
||||
_, err = f.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
hd, _, err := car.ReadHeader(bufio.NewReader(f))
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, hd.Version)
|
||||
|
||||
require.NoError(t, f.Close())
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
func genTmpFile(t *testing.T) string {
|
||||
f, err := os.CreateTemp("", "")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
func genNormalInputFile(t *testing.T) (filepath string, contents []byte) {
|
||||
s := strings.Repeat("abcde", 100)
|
||||
tmp, err := os.CreateTemp("", "")
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(tmp, strings.NewReader(s))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tmp.Close())
|
||||
return tmp.Name(), []byte(s)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user