Add tests cementing the DM-level behavior on simple paths
This commit is contained in:
Peter Rabbitson
parent
210485d800
commit
46937a1b9f
@ -805,6 +805,11 @@ workflows:
|
||||
suite: itest-deals_padding
|
||||
target: "./itests/deals_padding_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_partial_retrieval_dm-level
|
||||
suite: itest-deals_partial_retrieval_dm-level
|
||||
target: "./itests/deals_partial_retrieval_dm-level_test.go"
|
||||
|
||||
- test:
|
||||
name: test-itest-deals_partial_retrieval
|
||||
suite: itest-deals_partial_retrieval
|
||||
|
||||
252
itests/deals_partial_retrieval_dm-level_test.go
Normal file
252
itests/deals_partial_retrieval_dm-level_test.go
Normal file
@ -0,0 +1,252 @@
|
||||
package itests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/filecoin-project/go-fil-markets/storagemarket"
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/lotus/api"
|
||||
api0 "github.com/filecoin-project/lotus/api/v0api"
|
||||
"github.com/filecoin-project/lotus/chain/actors/policy"
|
||||
"github.com/filecoin-project/lotus/itests/kit"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipld/go-car"
|
||||
textselector "github.com/ipld/go-ipld-selector-text-lite"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// please talk to @ribasushi or @mikeal before modifying these test: there are
|
||||
// downstream dependencies on ADL-less operation
|
||||
var (
|
||||
adlFixtureCar = "fixtures/adl_test.car"
|
||||
adlFixtureRoot, _ = cid.Parse("bafybeiaigxwanoxyeuzyiknhrg6io6kobfbm37ozcips6qdwumub2gaomy")
|
||||
adlFixtureCommp, _ = cid.Parse("baga6ea4seaqjnmnrv4qsfz2rnda54mvo5al22dwpguhn2pmep63gl7bbqqqraai")
|
||||
adlFixturePieceSize = abi.PaddedPieceSize(1024)
|
||||
dmSelector = api.Selector("Links/0/Hash")
|
||||
dmTextSelector = textselector.Expression(dmSelector)
|
||||
dmExpectedResult = "NO ADL"
|
||||
dmExpectedCarBlockCount = 4
|
||||
dmDagSpec = []api.DagSpec{{DataSelector: &dmSelector}}
|
||||
)
|
||||
|
||||
func TestDMLevelPartialRetrieval(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
policy.SetPreCommitChallengeDelay(2)
|
||||
kit.QuietMiningLogs()
|
||||
client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs())
|
||||
dh := kit.NewDealHarness(t, client, miner, miner)
|
||||
ens.InterconnectAll().BeginMining(50 * time.Millisecond)
|
||||
|
||||
_, err := client.ClientImport(ctx, api.FileRef{Path: adlFixtureCar, IsCAR: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
caddr, err := client.WalletDefaultAddress(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
//
|
||||
// test retrieval from local car 1st
|
||||
require.NoError(t, testDMExportAsCar(
|
||||
ctx, client, api.ExportRef{
|
||||
FromLocalCAR: adlFixtureCar,
|
||||
Root: adlFixtureRoot,
|
||||
DAGs: dmDagSpec,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
require.NoError(t, testDMExportAsFile(
|
||||
ctx, client, api.ExportRef{
|
||||
FromLocalCAR: adlFixtureCar,
|
||||
Root: adlFixtureRoot,
|
||||
DAGs: dmDagSpec,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
|
||||
//
|
||||
// ensure V0 continues functioning as expected
|
||||
require.NoError(t, tesV0RetrievalAsCar(
|
||||
ctx, client, api0.RetrievalOrder{
|
||||
FromLocalCAR: adlFixtureCar,
|
||||
Root: adlFixtureRoot,
|
||||
DatamodelPathSelector: &dmTextSelector,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
require.NoError(t, testV0RetrievalAsFile(
|
||||
ctx, client, api0.RetrievalOrder{
|
||||
FromLocalCAR: adlFixtureCar,
|
||||
Root: adlFixtureRoot,
|
||||
DatamodelPathSelector: &dmTextSelector,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
|
||||
//
|
||||
// now perform a storage/retrieval deal as well, and retest
|
||||
dp := dh.DefaultStartDealParams()
|
||||
dp.Data = &storagemarket.DataRef{
|
||||
Root: adlFixtureRoot,
|
||||
PieceCid: &adlFixtureCommp,
|
||||
PieceSize: adlFixturePieceSize.Unpadded(),
|
||||
}
|
||||
proposalCid := dh.StartDeal(ctx, dp)
|
||||
|
||||
// Wait for the deal to reach StorageDealCheckForAcceptance on the client
|
||||
cd, err := client.ClientGetDealInfo(ctx, *proposalCid)
|
||||
require.NoError(t, err)
|
||||
require.Eventually(t, func() bool {
|
||||
cd, _ := client.ClientGetDealInfo(ctx, *proposalCid)
|
||||
return cd.State == storagemarket.StorageDealCheckForAcceptance
|
||||
}, 30*time.Second, 1*time.Second, "actual deal status is %s", storagemarket.DealStates[cd.State])
|
||||
|
||||
dh.WaitDealSealed(ctx, proposalCid, false, false, nil)
|
||||
|
||||
offers, err := client.ClientFindData(ctx, adlFixtureRoot, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, offers, "no offers")
|
||||
|
||||
retOrder := offers[0].Order(caddr)
|
||||
retOrder.DataSelector = &dmSelector
|
||||
|
||||
rr, err := client.ClientRetrieve(ctx, retOrder)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = client.ClientRetrieveWait(ctx, rr.DealID)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, testDMExportAsCar(
|
||||
ctx, client, api.ExportRef{
|
||||
DealID: rr.DealID,
|
||||
Root: adlFixtureRoot,
|
||||
DAGs: dmDagSpec,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
require.NoError(t, testDMExportAsFile(
|
||||
ctx, client, api.ExportRef{
|
||||
DealID: rr.DealID,
|
||||
Root: adlFixtureRoot,
|
||||
DAGs: dmDagSpec,
|
||||
}, t.TempDir(),
|
||||
))
|
||||
|
||||
}
|
||||
|
||||
func testDMExportAsFile(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error {
|
||||
out, err := ioutil.TempFile(tempDir, "exp-test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close() //nolint:errcheck
|
||||
|
||||
fileDest := api.FileRef{
|
||||
Path: out.Name(),
|
||||
}
|
||||
err = client.ClientExport(ctx, expDirective, fileDest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateDMUnixFile(out)
|
||||
}
|
||||
func testV0RetrievalAsFile(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error {
|
||||
out, err := ioutil.TempFile(tempDir, "exp-test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close() //nolint:errcheck
|
||||
|
||||
cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet
|
||||
err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{
|
||||
Path: out.Name(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateDMUnixFile(out)
|
||||
}
|
||||
func validateDMUnixFile(r io.Reader) error {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(data) != dmExpectedResult {
|
||||
return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func testDMExportAsCar(ctx context.Context, client *kit.TestFullNode, expDirective api.ExportRef, tempDir string) error {
|
||||
out, err := ioutil.TempFile(tempDir, "exp-test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close() //nolint:errcheck
|
||||
|
||||
carDest := api.FileRef{
|
||||
IsCAR: true,
|
||||
Path: out.Name(),
|
||||
}
|
||||
err = client.ClientExport(ctx, expDirective, carDest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return validateDMCar(out)
|
||||
}
|
||||
func tesV0RetrievalAsCar(ctx context.Context, client *kit.TestFullNode, retOrder api0.RetrievalOrder, tempDir string) error {
|
||||
out, err := ioutil.TempFile(tempDir, "exp-test")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close() //nolint:errcheck
|
||||
|
||||
cv0 := &api0.WrapperV1Full{client.FullNode} //nolint:govet
|
||||
err = cv0.ClientRetrieve(ctx, retOrder, &api.FileRef{
|
||||
Path: out.Name(),
|
||||
IsCAR: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return validateDMCar(out)
|
||||
}
|
||||
func validateDMCar(r io.Reader) error {
|
||||
cr, err := car.NewCarReader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(cr.Header.Roots) != 1 {
|
||||
return fmt.Errorf("expected a single root in result car, got %d", len(cr.Header.Roots))
|
||||
} else if cr.Header.Roots[0].String() != adlFixtureRoot.String() {
|
||||
return fmt.Errorf("expected root cid '%s', got '%s'", adlFixtureRoot.String(), cr.Header.Roots[0].String())
|
||||
}
|
||||
|
||||
blks := make([]blocks.Block, 0)
|
||||
for {
|
||||
b, err := cr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blks = append(blks, b)
|
||||
}
|
||||
|
||||
if len(blks) != dmExpectedCarBlockCount {
|
||||
return fmt.Errorf("expected a car file with %d blocks, got one with %d instead", dmExpectedCarBlockCount, len(blks))
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s%s", blks[2].RawData(), blks[3].RawData())
|
||||
if data != dmExpectedResult {
|
||||
return fmt.Errorf("retrieved data mismatch: expected '%s' got '%s'", dmExpectedResult, data)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
BIN
itests/fixtures/adl_test.car
Normal file
BIN
itests/fixtures/adl_test.car
Normal file
Binary file not shown.
@ -966,10 +966,6 @@ func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car
|
||||
}
|
||||
|
||||
dserv := merkledag.NewDAGService(blockservice.New(retrievalBs, offline.Exchange(retrievalBs)))
|
||||
roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv, !car)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing dag spec: %w", err)
|
||||
}
|
||||
|
||||
// Are we outputting a CAR?
|
||||
if car {
|
||||
@ -978,6 +974,22 @@ func (a *API) ClientExportInto(ctx context.Context, exportRef api.ExportRef, car
|
||||
return carv2.ExtractV1File(carPath, dest.Path)
|
||||
}
|
||||
|
||||
// if this is a path-selector, the user expects the car to start from the
|
||||
// root they asked for ( full merkle proof, no heuristic )
|
||||
if len(exportRef.DAGs) == 1 && exportRef.DAGs[0].DataSelector != nil && !strings.HasPrefix(string(*exportRef.DAGs[0].DataSelector), "{") {
|
||||
sel, err := getDataSelector(exportRef.DAGs[0].DataSelector)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing dag spec: %w", err)
|
||||
}
|
||||
return a.outputCAR(ctx, []dagSpec{{root: exportRef.Root, selector: sel}}, retrievalBs, dest)
|
||||
}
|
||||
}
|
||||
|
||||
roots, err := parseDagSpec(ctx, exportRef.Root, exportRef.DAGs, dserv, !car)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("parsing dag spec: %w", err)
|
||||
}
|
||||
if car {
|
||||
return a.outputCAR(ctx, roots, retrievalBs, dest)
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user