cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #7673 from filecoin-project/chore/DM-level-tests_plus_merkle-proof-cars

Browse Source 
Chore/dm level tests plus merkle proof cars
This commit is contained in:
Łukasz Magiera 2021-11-24 20:43:39 +01:00 committed by GitHub
commit 0c884734a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 278 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.circleci/config.yml
View File

@ -805,6 +805,11 @@ workflows:
suite: itest-deals_padding
target: "./itests/deals_padding_test.go"
- test:
name: test-itest-deals_partial_retrieval_dm-level
suite: itest-deals_partial_retrieval_dm-level
target: "./itests/deals_partial_retrieval_dm-level_test.go"
- test:
name: test-itest-deals_partial_retrieval
suite: itest-deals_partial_retrieval

252
itests/deals_partial_retrieval_dm-level_test.go Normal file
View File

@ -0,0 +1,252 @@
package itests
import (
"context"
"fmt"
"io"
"io/ioutil"
"testing"
"time"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
api0 "github.com/filecoin-project/lotus/api/v0api"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/itests/kit"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipld/go-car"
textselector "github.com/ipld/go-ipld-selector-text-lite"
"github.com/stretchr/testify/require"
)
// please talk to @ribasushi or @mikeal before modifying these test: there are
// downstream dependencies on ADL-less operation
var (
adlFixtureCar = "fixtures/adl_test.car"
adlFixtureRoot, _ = cid.Parse("bafybeiaigxwanoxyeuzyiknhrg6io6kobfbm37ozcips6qdwumub2gaomy")
adlFixtureCommp, _ = cid.Parse("baga6ea4seaqjnmnrv4qsfz2rnda54mvo5al22dwpguhn2pmep63gl7bbqqqraai")
adlFixturePieceSize = abi.PaddedPieceSize(1024)
dmSelector = api.Selector("Links/0/Hash")
dmTextSelector = textselector.Expression(dmSelector)
dmExpectedResult = "NO ADL"
dmExpectedCarBlockCount = 4
dmDagSpec = []api.DagSpec{{DataSelector: &dmSelector}}
)
func TestDMLevelPartialRetrieval(t *testing.T) {
ctx := context.Background()
policy.SetPreCommitChallengeDelay(2)
kit.QuietMiningLogs()
client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs())
dh := kit.NewDealHarness(t, client, miner, miner)
ens.InterconnectAll().BeginMining(50 * time.Millisecond)
_, err := client.ClientImport(ctx, api.FileRef{Path: adlFixtureCar, IsCAR: true})
require.NoError(t, err)
caddr, err := client.WalletDefaultAddress(ctx)
require.NoError(t, err)
//
// test retrieval from local car 1st
require.NoError(t, testDMExportAsCar(
ctx, client, api.ExportRef{
FromLocalCAR: adlFixtureCar,
Root: adlFixtureRoot,
DAGs: dmDagSpec,
}, t.TempDir(),
))
require.NoError(t, testDMExportAsFile(
ctx, client, api.ExportRef{
FromLocalCAR: adlFixtureCar,
Root: adlFixtureRoot,
DAGs: dmDagSpec,
}, t.TempDir(),
))
//
// ensure V0 continues functioning as expected
require.NoError(t, tesV0RetrievalAsCar(
ctx, client, api0.RetrievalOrder{
FromLocalCAR: adlFixtureCar,
Root: adlFixtureRoot,
DatamodelPathSelector: &dmTextSelector,
}, t.TempDir(),
))
require.NoError(t, testV0RetrievalAsFile(
ctx, client, api0.RetrievalOrder{
FromLocalCAR: adlFixtureCar,
Root: adlFixtureRoot,
DatamodelPathSelector: &dmTextSelector,
}, t.TempDir(),
))
//
// now perform a storage/retrieval deal as well, and retest
dp := dh.DefaultStartDealParams()
dp.Data = &storagemarket.DataRef{
Root: adlFixtureRoot,
PieceCid: &adlFixtureCommp,
PieceSize: adlFixturePieceSize.Unpadded(),
}
proposalCid := dh.StartDeal(ctx, dp)
// Wait for the deal to reach StorageDealCheckForAcceptance on the client
cd, err := client.ClientGetDealInfo(ctx, *proposalCid)
require.NoError(t, err)
require.Eventually(t, func() bool {
cd, _ := client.ClientGetDealInfo(ctx, *proposalCid)
return cd.State == storagemarket.StorageDealCheckForAcceptance
}, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State])
dh.WaitDealSealed(ctx, proposalCid, false, false, nil)
offers, err := client.ClientFindData(ctx, adlFixtureRoot, nil)
require.NoError(t, err)
require.NotEmpty(t, offers, "no offers")
retOrder := offers[0].Order(caddr)
retOrder.DataSelector = &dmSelector
rr, err := client.ClientRetrieve(ctx, retOrder)
require.NoError(t, err)
err = client.ClientRetrieveWait(ctx, rr.DealID)
require.NoError(t, err)
require.NoError(t, testDMExportAsCar(
ctx, client, api.ExportRef{
DealID: rr.DealID,
Root: adlFixtureRoot,
DAGs: dmDagSpec,
}, t.TempDir(),
))
require.NoError(t, testDMExportAsFile(
ctx, client, api.ExportRef{
DealID: rr.DealID,
Root: adlFixtureRoot,
DAGs: dmDagSpec,
}, t.TempDir(),
))
}
func testDMExportAsFile(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error {
out, err := ioutil.TempFile(tempDir, "exp-test")
if err != nil {
return err
}
defer out.Close() //nolint:errcheck
fileDest := api.FileRef{
Path: out.Name(),
}
err = client.ClientExport(ctx, expDirective, fileDest)
if err != nil {
return err
}
return validateDMUnixFile(out)
}
func testV0RetrievalAsFile(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error {
out, err := ioutil.TempFile(tempDir, "exp-test")
if err != nil {
return err
}
defer out.Close() //nolint:errcheck
cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet
err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{
Path: out.Name(),
})
if err != nil {
return err
}
return validateDMUnixFile(out)
}
func validateDMUnixFile(r io.Reader) error {
data, err := io.ReadAll(r)
if err != nil {
return err
}
if string(data) != dmExpectedResult {
return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data)
}
return nil
}
func testDMExportAsCar(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error {
out, err := ioutil.TempFile(tempDir, "exp-test")
if err != nil {
return err
}
defer out.Close() //nolint:errcheck
carDest := api.FileRef{
IsCAR: true,
Path: out.Name(),
}
err = client.ClientExport(ctx, expDirective, carDest)
if err != nil {
return err
}
return validateDMCar(out)
}
func tesV0RetrievalAsCar(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error {
out, err := ioutil.TempFile(tempDir, "exp-test")
if err != nil {
return err
}
defer out.Close() //nolint:errcheck
cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet
err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{
Path: out.Name(),
IsCAR: true,
})
if err != nil {
return err
}
return validateDMCar(out)
}
func validateDMCar(r io.Reader) error {
cr, err := car.NewCarReader(r)
if err != nil {
return err
}
if len(cr.Header.Roots) != 1 {
return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots))
} else if cr.Header.Roots[0].String() != adlFixtureRoot.String() {
return fmt.Errorf("expected root cid '%s', got '%s'", adlFixtureRoot.String(), cr.Header.Roots[0].String())
}
blks := make([]blocks.Block, 0)
for {
b, err := cr.Next()
if err == io.EOF {
break
} else if err != nil {
return err
}
blks = append(blks, b)
}
if len(blks) != dmExpectedCarBlockCount {
return fmt.Errorf("expected a car file with %d blocks, got one with %d instead", dmExpectedCarBlockCount, len(blks))
}
data := fmt.Sprintf("%s%s", blks[2].RawData(), blks[3].RawData())
if data != dmExpectedResult {
return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data)
}
return nil
}

14
itests/deals_partial_retrieval_test.go
View File

@ -31,7 +31,6 @@ var (
carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina")
carPieceSize = abi.PaddedPieceSize(2097152)
textSelector = api.Selector("8/1/8/1/0/1/0")
storPowCid, _ = cid.Parse("bafkqaetgnfwc6mjpon2g64tbm5sxa33xmvza")
textSelectorNonLink = api.Selector("8/1/8/1/0/1")
textSelectorNonexistent = api.Selector("42")
expectedResult = "fil/1/storagepower"
@ -115,7 +114,6 @@ func TestPartialRetrieval(t *testing.T) {
Path: outFile.Name(),
IsCAR: retrieveAsCar,
},
storPowCid,
outFile,
))
@ -145,7 +143,6 @@ func TestPartialRetrieval(t *testing.T) {
DAGs: []api.DagSpec{{DataSelector: &textSelectorNonexistent}},
},
&api.FileRef{},
storPowCid,
nil,
),
fmt.Sprintf("parsing dag spec: path selection does not match a node within %s", carRoot),
@ -167,14 +164,13 @@ func TestPartialRetrieval(t *testing.T) {
DAGs: []api.DagSpec{{DataSelector: &textSelectorNonLink}},
},
&api.FileRef{},
storPowCid,
nil,
),
fmt.Sprintf("parsing dag spec: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink),
)
}
func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, expRootCid cid.Cid, outFile *os.File) error {
func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, outFile *os.File) error {
if retOrder.Total.Nil() {
retOrder.Total = big.Zero()
@ -217,7 +213,7 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde
if len(cr.Header.Roots) != 1 {
return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots))
} else if cr.Header.Roots[0].String() != expRootCid.String() {
} else if cr.Header.Roots[0].String() != carRoot.String() {
return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String())
}
@ -233,11 +229,11 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde
blks = append(blks, b)
}
if len(blks) != 1 {
return fmt.Errorf("expected a car file with 1 blocks, got one with %d instead", len(blks))
if len(blks) != 3 {
return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks))
}
data = blks[0].RawData()
data = blks[2].RawData()
}
if string(data) != expectedResult {

BIN
itests/fixtures/adl_test.car Normal file
View File

Binary file not shown.

20
node/impl/client/client.go
View File

@ -966,10 +966,6 @@ func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car
}
dserv := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs)))
roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv, !car)
if err != nil {
return xerrors.Errorf("parsing dag spec: %w", err)
}
// Are we outputting a CAR?
if car {
@ -978,6 +974,22 @@ func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car
return carv2.ExtractV1File(carPath, dest.Path)
}
// if this is a path-selector, the user expects the car to start from the
// root they asked for ( full merkle proof, no heuristic )
if len(exportRef.DAGs) == 1 && exportRef.DAGs[0].DataSelector != nil && !strings.HasPrefix(string(*exportRef.DAGs[0].DataSelector), "{") {
sel, err := getDataSelector(exportRef.DAGs[0].DataSelector)
if err != nil {
return xerrors.Errorf("parsing dag spec: %w", err)
}
return a.outputCAR(ctx, []dagSpec{{root: exportRef.Root, selector: sel}}, retrievalBs, dest)
}
}
roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv, !car)
if err != nil {
return xerrors.Errorf("parsing dag spec: %w", err)
}
if car {
return a.outputCAR(ctx, roots, retrievalBs, dest)
}