From 46937a1b9fc794609b97655436903f3b66a6113c Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Wed, 24 Nov 2021 15:08:36 +0100 Subject: [PATCH 1/2] Add tests cementing the DM-level behavior on simple paths --- .circleci/config.yml | 5 + .../deals_partial_retrieval_dm-level_test.go | 252 ++++++++++++++++++ itests/fixtures/adl_test.car | Bin 0 -> 597 bytes node/impl/client/client.go | 20 +- 4 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 itests/deals_partial_retrieval_dm-level_test.go create mode 100644 itests/fixtures/adl_test.car diff --git a/.circleci/config.yml b/.circleci/config.yml index 30f2d5c01..e74cb7736 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -805,6 +805,11 @@ workflows: suite: itest-deals_padding target: "./itests/deals_padding_test.go" + - test: + name: test-itest-deals_partial_retrieval_dm-level + suite: itest-deals_partial_retrieval_dm-level + target: "./itests/deals_partial_retrieval_dm-level_test.go" + - test: name: test-itest-deals_partial_retrieval suite: itest-deals_partial_retrieval diff --git a/itests/deals_partial_retrieval_dm-level_test.go b/itests/deals_partial_retrieval_dm-level_test.go new file mode 100644 index 000000000..88c2a0d6e --- /dev/null +++ b/itests/deals_partial_retrieval_dm-level_test.go @@ -0,0 +1,252 @@ +package itests + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "testing" + "time" + + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + api0 "github.com/filecoin-project/lotus/api/v0api" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/itests/kit" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car" + textselector "github.com/ipld/go-ipld-selector-text-lite" + "github.com/stretchr/testify/require" +) + +// please talk to @ribasushi or @mikeal before modifying these test: there are +// downstream dependencies on ADL-less operation +var ( + adlFixtureCar = "fixtures/adl_test.car" + adlFixtureRoot, _ = cid.Parse("bafybeiaigxwanoxyeuzyiknhrg6io6kobfbm37ozcips6qdwumub2gaomy") + adlFixtureCommp, _ = cid.Parse("baga6ea4seaqjnmnrv4qsfz2rnda54mvo5al22dwpguhn2pmep63gl7bbqqqraai") + adlFixturePieceSize = abi.PaddedPieceSize(1024) + dmSelector = api.Selector("Links/0/Hash") + dmTextSelector = textselector.Expression(dmSelector) + dmExpectedResult = "NO ADL" + dmExpectedCarBlockCount = 4 + dmDagSpec = []api.DagSpec{{DataSelector: &dmSelector}} +) + +func TestDMLevelPartialRetrieval(t *testing.T) { + + ctx := context.Background() + + policy.SetPreCommitChallengeDelay(2) + kit.QuietMiningLogs() + client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs()) + dh := kit.NewDealHarness(t, client, miner, miner) + ens.InterconnectAll().BeginMining(50 * time.Millisecond) + + _, err := client.ClientImport(ctx, api.FileRef{Path: adlFixtureCar, IsCAR: true}) + require.NoError(t, err) + + caddr, err := client.WalletDefaultAddress(ctx) + require.NoError(t, err) + + // + // test retrieval from local car 1st + require.NoError(t, testDMExportAsCar( + ctx, client, api.ExportRef{ + FromLocalCAR: adlFixtureCar, + Root: adlFixtureRoot, + DAGs: dmDagSpec, + }, t.TempDir(), + )) + require.NoError(t, testDMExportAsFile( + ctx, client, api.ExportRef{ + FromLocalCAR: adlFixtureCar, + Root: adlFixtureRoot, + DAGs: dmDagSpec, + }, t.TempDir(), + )) + + // + // ensure V0 continues functioning as expected + require.NoError(t, tesV0RetrievalAsCar( + ctx, client, api0.RetrievalOrder{ + FromLocalCAR: adlFixtureCar, + Root: adlFixtureRoot, + DatamodelPathSelector: &dmTextSelector, + }, t.TempDir(), + )) + require.NoError(t, testV0RetrievalAsFile( + ctx, client, api0.RetrievalOrder{ + FromLocalCAR: adlFixtureCar, + Root: adlFixtureRoot, + DatamodelPathSelector: &dmTextSelector, + }, t.TempDir(), + )) + + // + // now perform a storage/retrieval deal as well, and retest + dp := dh.DefaultStartDealParams() + dp.Data = &storagemarket.DataRef{ + Root: adlFixtureRoot, + PieceCid: &adlFixtureCommp, + PieceSize: adlFixturePieceSize.Unpadded(), + } + proposalCid := dh.StartDeal(ctx, dp) + + // Wait for the deal to reach StorageDealCheckForAcceptance on the client + cd, err := client.ClientGetDealInfo(ctx, *proposalCid) + require.NoError(t, err) + require.Eventually(t, func() bool { + cd, _ := client.ClientGetDealInfo(ctx, *proposalCid) + return cd.State == storagemarket.StorageDealCheckForAcceptance + }, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State]) + + dh.WaitDealSealed(ctx, proposalCid, false, false, nil) + + offers, err := client.ClientFindData(ctx, adlFixtureRoot, nil) + require.NoError(t, err) + require.NotEmpty(t, offers, "no offers") + + retOrder := offers[0].Order(caddr) + retOrder.DataSelector = &dmSelector + + rr, err := client.ClientRetrieve(ctx, retOrder) + require.NoError(t, err) + + err = client.ClientRetrieveWait(ctx, rr.DealID) + require.NoError(t, err) + + require.NoError(t, testDMExportAsCar( + ctx, client, api.ExportRef{ + DealID: rr.DealID, + Root: adlFixtureRoot, + DAGs: dmDagSpec, + }, t.TempDir(), + )) + require.NoError(t, testDMExportAsFile( + ctx, client, api.ExportRef{ + DealID: rr.DealID, + Root: adlFixtureRoot, + DAGs: dmDagSpec, + }, t.TempDir(), + )) + +} + +func testDMExportAsFile(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error { + out, err := ioutil.TempFile(tempDir, "exp-test") + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + + fileDest := api.FileRef{ + Path: out.Name(), + } + err = client.ClientExport(ctx, expDirective, fileDest) + if err != nil { + return err + } + return validateDMUnixFile(out) +} +func testV0RetrievalAsFile(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error { + out, err := ioutil.TempFile(tempDir, "exp-test") + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + + cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet + err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{ + Path: out.Name(), + }) + if err != nil { + return err + } + return validateDMUnixFile(out) +} +func validateDMUnixFile(r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + if string(data) != dmExpectedResult { + return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data) + } + + return nil +} + +func testDMExportAsCar(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error { + out, err := ioutil.TempFile(tempDir, "exp-test") + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + + carDest := api.FileRef{ + IsCAR: true, + Path: out.Name(), + } + err = client.ClientExport(ctx, expDirective, carDest) + if err != nil { + return err + } + + return validateDMCar(out) +} +func tesV0RetrievalAsCar(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error { + out, err := ioutil.TempFile(tempDir, "exp-test") + if err != nil { + return err + } + defer out.Close() //nolint:errcheck + + cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet + err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{ + Path: out.Name(), + IsCAR: true, + }) + if err != nil { + return err + } + + return validateDMCar(out) +} +func validateDMCar(r io.Reader) error { + cr, err := car.NewCarReader(r) + if err != nil { + return err + } + + if len(cr.Header.Roots) != 1 { + return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) + } else if cr.Header.Roots[0].String() != adlFixtureRoot.String() { + return fmt.Errorf("expected root cid '%s', got '%s'", adlFixtureRoot.String(), cr.Header.Roots[0].String()) + } + + blks := make([]blocks.Block, 0) + for { + b, err := cr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + blks = append(blks, b) + } + + if len(blks) != dmExpectedCarBlockCount { + return fmt.Errorf("expected a car file with %d blocks, got one with %d instead", dmExpectedCarBlockCount, len(blks)) + } + + data := fmt.Sprintf("%s%s", blks[2].RawData(), blks[3].RawData()) + if data != dmExpectedResult { + return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data) + } + + return nil +} diff --git a/itests/fixtures/adl_test.car b/itests/fixtures/adl_test.car new file mode 100644 index 0000000000000000000000000000000000000000..d00ca091519d963ea018c24a1dc722489c0c9746 GIT binary patch literal 597 zcmcColv>hxh=Qu^)V|Br#tdfvO-qR6ke>*^yolt?*ayU7r|#B zwOkw=OcHDg%nHov#2D}AuK-p@$Y@6wpHNT;5)27#NQg}nd8wD-*t+) Date: Wed, 24 Nov 2021 15:10:13 +0100 Subject: [PATCH 2/2] Modify preexisting test to reflect the simple-path car export behavior I am open to having a flag or something, as long as the behavior reflected in deals_partial_retrieval_dm-level_test.go can be preserved for v0, and opt-in for v1 --- itests/deals_partial_retrieval_test.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/itests/deals_partial_retrieval_test.go b/itests/deals_partial_retrieval_test.go index bb9611594..21aede286 100644 --- a/itests/deals_partial_retrieval_test.go +++ b/itests/deals_partial_retrieval_test.go @@ -31,7 +31,6 @@ var ( carCommp, _ = cid.Parse("baga6ea4seaqmrivgzei3fmx5qxtppwankmtou6zvigyjaveu3z2zzwhysgzuina") carPieceSize = abi.PaddedPieceSize(2097152) textSelector = api.Selector("8/1/8/1/0/1/0") - storPowCid, _ = cid.Parse("bafkqaetgnfwc6mjpon2g64tbm5sxa33xmvza") textSelectorNonLink = api.Selector("8/1/8/1/0/1") textSelectorNonexistent = api.Selector("42") expectedResult = "fil/1/storagepower" @@ -115,7 +114,6 @@ func TestPartialRetrieval(t *testing.T) { Path: outFile.Name(), IsCAR: retrieveAsCar, }, - storPowCid, outFile, )) @@ -145,7 +143,6 @@ func TestPartialRetrieval(t *testing.T) { DAGs: []api.DagSpec{{DataSelector: &textSelectorNonexistent}}, }, &api.FileRef{}, - storPowCid, nil, ), fmt.Sprintf("parsing dag spec: path selection does not match a node within %s", carRoot), @@ -167,14 +164,13 @@ func TestPartialRetrieval(t *testing.T) { DAGs: []api.DagSpec{{DataSelector: &textSelectorNonLink}}, }, &api.FileRef{}, - storPowCid, nil, ), fmt.Sprintf("parsing dag spec: error while locating partial retrieval sub-root: unsupported selection path '%s' does not correspond to a block boundary (a.k.a. CID link)", textSelectorNonLink), ) } -func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, expRootCid cid.Cid, outFile *os.File) error { +func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrder api.RetrievalOrder, eref api.ExportRef, retRef *api.FileRef, outFile *os.File) error { if retOrder.Total.Nil() { retOrder.Total = big.Zero() @@ -217,7 +213,7 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde if len(cr.Header.Roots) != 1 { return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots)) - } else if cr.Header.Roots[0].String() != expRootCid.String() { + } else if cr.Header.Roots[0].String() != carRoot.String() { return fmt.Errorf("expected root cid '%s', got '%s'", carRoot.String(), cr.Header.Roots[0].String()) } @@ -233,11 +229,11 @@ func testGenesisRetrieval(ctx context.Context, client *kit.TestFullNode, retOrde blks = append(blks, b) } - if len(blks) != 1 { - return fmt.Errorf("expected a car file with 1 blocks, got one with %d instead", len(blks)) + if len(blks) != 3 { + return fmt.Errorf("expected a car file with 3 blocks, got one with %d instead", len(blks)) } - data = blks[0].RawData() + data = blks[2].RawData() } if string(data) != expectedResult {