From 71ddf73f4492e4ca34e0028727d6e6a1706664bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sat, 10 Jul 2021 10:48:18 +0100 Subject: [PATCH 1/9] check if merging ipfs/go-graphsync#177 fixes deal concurrency. --- go.mod | 2 +- go.sum | 2 ++ itests/deals_concurrent_test.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2bd53ea59..2c223b342 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.6.1 + github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 8fb1d54d3..52d3a2908 100644 --- a/go.sum +++ b/go.sum @@ -628,6 +628,8 @@ github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZ github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= github.com/ipfs/go-graphsync v0.6.1 h1:i9wN7YkBXWwIsUjVQeuaDxFB59yWZrG1xL564Nz7aGE= github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= +github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a h1:EgOVzYGldAgzfb0OLrvCxRazAtXjjhgnPfYco+l3URU= +github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 44b25c7b3..fad034f2c 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -47,7 +47,7 @@ func TestDealCyclesConcurrent(t *testing.T) { } // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{1} + cycles := []int{1, 2, 4, 8} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) From cbf281d1cae299cbb938d9f92e0db7d18872f37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sat, 10 Jul 2021 10:49:37 +0100 Subject: [PATCH 2/9] go mod tidy. --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index 52d3a2908..730c29cb2 100644 --- a/go.sum +++ b/go.sum @@ -626,7 +626,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= -github.com/ipfs/go-graphsync v0.6.1 h1:i9wN7YkBXWwIsUjVQeuaDxFB59yWZrG1xL564Nz7aGE= github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a h1:EgOVzYGldAgzfb0OLrvCxRazAtXjjhgnPfYco+l3URU= github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= From b2feb7e96066201760fff03d5fbfa523b3882da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 16 Jul 2021 17:30:45 +0100 Subject: [PATCH 3/9] upgrade go-fil-markets, go-data-transfer; add logging. --- go.mod | 4 ++-- go.sum | 10 +++++----- itests/deals_concurrent_test.go | 29 ++++++++++++++++++++++++----- itests/kit/deals.go | 7 +++++++ 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 2c223b342..016fbea11 100644 --- a/go.mod +++ b/go.mod @@ -33,9 +33,9 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v1.6.0 + github.com/filecoin-project/go-data-transfer v1.7.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a - github.com/filecoin-project/go-fil-markets v1.5.0 + github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 730c29cb2..0b5bb2011 100644 --- a/go.sum +++ b/go.sum @@ -275,16 +275,16 @@ github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= -github.com/filecoin-project/go-data-transfer v1.6.0 h1:DHIzEc23ydRCCBwtFet3MfgO8gMpZEnw60Y+s71oX6o= -github.com/filecoin-project/go-data-transfer v1.6.0/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc= +github.com/filecoin-project/go-data-transfer v1.7.0 h1:mFRn+UuTdPROmhplLSekzd4rAs9ug8ubtSY4nw9wYkU= +github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRDG7uJvph22KGL2M4iOuF8EINaU= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.5.0 h1:3KEs01L8XFCEgujZ6ggFjr1XWjpjTQcmSSeo3I99I0k= -github.com/filecoin-project/go-fil-markets v1.5.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771 h1:RwAR9KQe8sgAsaGJjT3QvcdI/do9DzsZh+P81sc76cw= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -626,7 +626,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= -github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= +github.com/ipfs/go-graphsync v0.6.4/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a h1:EgOVzYGldAgzfb0OLrvCxRazAtXjjhgnPfYco+l3URU= github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index fad034f2c..6959fe5f1 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -24,6 +25,12 @@ func TestDealCyclesConcurrent(t *testing.T) { t.Skip("skipping test in short mode") } + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + kit.QuietMiningLogs() blockTime := 10 * time.Millisecond @@ -47,7 +54,7 @@ func TestDealCyclesConcurrent(t *testing.T) { } // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{1, 2, 4, 8} + cycles := []int{2} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) @@ -58,13 +65,19 @@ func TestDealCyclesConcurrent(t *testing.T) { } } -func TestSimultenousTransferLimit(t *testing.T) { +func TestSimultanenousTransferLimit(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } kit.QuietMiningLogs() + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + blockTime := 10 * time.Millisecond // For these tests where the block time is artificially short, just use @@ -72,9 +85,10 @@ func TestSimultenousTransferLimit(t *testing.T) { // so that the deal starts sealing in time startEpoch := abi.ChainEpoch(2 << 12) + const graphsyncThrottle = 2 runTest := func(t *testing.T) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts( - node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(2))), + node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))), )) ens.InterconnectAll().BeginMining(blockTime) dh := kit.NewDealHarness(t, client, miner) @@ -111,16 +125,21 @@ func TestSimultenousTransferLimit(t *testing.T) { } }() + const concurrency = 10 + t.Logf("running concurrent deals: %d", concurrency) + dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ - N: 1, // TODO: set to 20 after https://github.com/ipfs/go-graphsync/issues/175 is fixed + N: concurrency, // TODO: set to 20 after https://github.com/ipfs/go-graphsync/issues/175 is fixed FastRetrieval: true, StartEpoch: startEpoch, }) + t.Logf("all deals finished") + cancel() wg.Wait() - require.LessOrEqual(t, maxOngoing, 2) + require.LessOrEqual(t, maxOngoing, graphsyncThrottle) } runTest(t) diff --git a/itests/kit/deals.go b/itests/kit/deals.go index d9129b76a..793dd52ba 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -290,6 +290,7 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) { for i := 0; i < opts.N; i++ { i := i errgrp.Go(func() (err error) { + defer dh.t.Logf("finished concurrent deal %d/%d", i, opts.N) defer func() { // This is necessary because golang can't deal with test // failures being reported from children goroutines ¯\_(ツ)_/¯ @@ -297,11 +298,17 @@ func (dh *DealHarness) RunConcurrentDeals(opts RunConcurrentDealsOpts) { err = fmt.Errorf("deal failed: %s", r) } }() + + dh.t.Logf("making storage deal %d/%d", i, opts.N) + deal, res, inPath := dh.MakeOnlineDeal(context.Background(), MakeFullDealParams{ Rseed: 5 + i, FastRet: opts.FastRetrieval, StartEpoch: opts.StartEpoch, }) + + dh.t.Logf("retrieving deal %d/%d", i, opts.N) + outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, opts.CarExport) AssertFilesEqual(dh.t, inPath, outPath) return nil From aeb8e57eef1f774d022247a21db5379367ceb584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 16 Jul 2021 18:17:34 +0100 Subject: [PATCH 4/9] use https://github.com/ipfs/go-graphsync/pull/176. --- go.mod | 4 ++-- go.sum | 14 ++++++++++++-- itests/deals_concurrent_test.go | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 016fbea11..489c4c924 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.7.0 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a - github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771 + github.com/filecoin-project/go-fil-markets v1.6.0-rc1 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 @@ -77,7 +77,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a + github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 0b5bb2011..00dc102f4 100644 --- a/go.sum +++ b/go.sum @@ -275,14 +275,21 @@ github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= +github.com/filecoin-project/go-data-transfer v1.6.0/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc= github.com/filecoin-project/go-data-transfer v1.7.0 h1:mFRn+UuTdPROmhplLSekzd4rAs9ug8ubtSY4nw9wYkU= github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRDG7uJvph22KGL2M4iOuF8EINaU= +github.com/filecoin-project/go-data-transfer v1.7.1-0.20210702215557-2b03c2747d90 h1:4XA1a9soBdENCv/rs/kM3Lvk76zhFhF68fEEhy3+GD0= +github.com/filecoin-project/go-data-transfer v1.7.1-0.20210702215557-2b03c2747d90/go.mod h1:TGV2vqXsGcoK6EHY6HTZFiXB1ozU0rUkiT1Hrx+6j90= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= +github.com/filecoin-project/go-fil-markets v1.5.0 h1:3KEs01L8XFCEgujZ6ggFjr1XWjpjTQcmSSeo3I99I0k= +github.com/filecoin-project/go-fil-markets v1.5.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps= +github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771 h1:RwAR9KQe8sgAsaGJjT3QvcdI/do9DzsZh+P81sc76cw= github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= @@ -626,9 +633,12 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= +github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-graphsync v0.6.4/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= -github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a h1:EgOVzYGldAgzfb0OLrvCxRazAtXjjhgnPfYco+l3URU= -github.com/ipfs/go-graphsync v0.6.5-0.20210621182231-957951706c7a/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= +github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7 h1:STKytbr5vpxe1a+sKXuI2ZU8KKplkiOkgkJgugXBfSM= +github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= +github.com/ipfs/go-graphsync v0.6.5-0.20210702215108-234bf651a8a7 h1:IIv1uDZYxWqQ1BIynIn+5MTYdxi2c2+O3GO+uq2t3Hw= +github.com/ipfs/go-graphsync v0.6.5-0.20210702215108-234bf651a8a7/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 6959fe5f1..2a3af6368 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -125,7 +125,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { } }() - const concurrency = 10 + const concurrency = 20 t.Logf("running concurrent deals: %d", concurrency) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ From 089bcceb33704628a98e51be65d0835f63107300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Fri, 16 Jul 2021 19:09:56 +0100 Subject: [PATCH 5/9] remove test concurrency guardrails. --- itests/deals_concurrent_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 716be04e1..b9920ba5a 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -57,8 +56,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { }) } - // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{1} + cycles := []int{4, 8} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) @@ -102,8 +100,7 @@ func TestDealCyclesConcurrent(t *testing.T) { }) } - // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{2} + cycles := []int{1, 2, 4, 8} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) @@ -178,7 +175,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { t.Logf("running concurrent deals: %d", concurrency) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ - N: concurrency, // TODO: set to 20 after https://github.com/ipfs/go-graphsync/issues/175 is fixed + N: concurrency, FastRetrieval: true, StartEpoch: startEpoch, }) From 3b4df5e10df1bd59f39eb7472358e58d4fa91e8d Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 16 Jul 2021 11:46:03 -0700 Subject: [PATCH 6/9] fix(itests): fix simultaneous transfer logic limit parallelism on client side and also fix the issue with how ongoing transfers are detected --- itests/deals_concurrent_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 716be04e1..56b3f7b15 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -14,7 +14,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/modules" @@ -138,6 +137,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { runTest := func(t *testing.T) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts( node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))), + node.Override(new(dtypes.Graphsync), modules.Graphsync(graphsyncThrottle)), )) ens.InterconnectAll().BeginMining(blockTime) dh := kit.NewDealHarness(t, client, miner, miner) @@ -159,7 +159,7 @@ func TestSimultanenousTransferLimit(t *testing.T) { select { case u := <-du: t.Logf("%d - %s", u.TransferID, datatransfer.Statuses[u.Status]) - if u.Status == datatransfer.Ongoing { + if u.Status == datatransfer.Ongoing && u.Transferred > 0 { ongoing[u.TransferID] = struct{}{} } else { delete(ongoing, u.TransferID) From c1b304b633b9de33616a3935fbadb15b25aba80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 20 Jul 2021 15:43:44 +0100 Subject: [PATCH 7/9] update graphsync. --- go.mod | 2 +- go.sum | 14 ++------------ itests/deals_concurrent_test.go | 16 ++++++++-------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index b9cfa266b..493ba9219 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7 + github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index 9c98edaa9..e345cd93c 100644 --- a/go.sum +++ b/go.sum @@ -275,23 +275,16 @@ github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= -github.com/filecoin-project/go-data-transfer v1.6.0/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc= github.com/filecoin-project/go-data-transfer v1.7.0 h1:mFRn+UuTdPROmhplLSekzd4rAs9ug8ubtSY4nw9wYkU= github.com/filecoin-project/go-data-transfer v1.7.0/go.mod h1:GLRr5BmLEqsLwXfiRDG7uJvph22KGL2M4iOuF8EINaU= -github.com/filecoin-project/go-data-transfer v1.7.1-0.20210702215557-2b03c2747d90 h1:4XA1a9soBdENCv/rs/kM3Lvk76zhFhF68fEEhy3+GD0= -github.com/filecoin-project/go-data-transfer v1.7.1-0.20210702215557-2b03c2747d90/go.mod h1:TGV2vqXsGcoK6EHY6HTZFiXB1ozU0rUkiT1Hrx+6j90= github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ= github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.5.0 h1:3KEs01L8XFCEgujZ6ggFjr1XWjpjTQcmSSeo3I99I0k= -github.com/filecoin-project/go-fil-markets v1.5.0/go.mod h1:7be6zzFwaN8kxVeYZf/UUj/JilHC0ogPvWqE1TW8Ptk= github.com/filecoin-project/go-fil-markets v1.6.0-rc1 h1:kQtND2NXz/cfGkjq+f5MCtz2oZAQabQvQ/zu4fppIps= github.com/filecoin-project/go-fil-markets v1.6.0-rc1/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= -github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771 h1:RwAR9KQe8sgAsaGJjT3QvcdI/do9DzsZh+P81sc76cw= -github.com/filecoin-project/go-fil-markets v1.6.0-rc1.0.20210715093612-b9e514cf3771/go.mod h1:S/C9PcSLFp75NpaF5aUqutnhXVJk6hM2dhWPYNq2jCQ= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -633,12 +626,9 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE= github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= -github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= github.com/ipfs/go-graphsync v0.6.4/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= -github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7 h1:STKytbr5vpxe1a+sKXuI2ZU8KKplkiOkgkJgugXBfSM= -github.com/ipfs/go-graphsync v0.6.5-0.20210702211749-5341e6add3e7/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= -github.com/ipfs/go-graphsync v0.6.5-0.20210702215108-234bf651a8a7 h1:IIv1uDZYxWqQ1BIynIn+5MTYdxi2c2+O3GO+uq2t3Hw= -github.com/ipfs/go-graphsync v0.6.5-0.20210702215108-234bf651a8a7/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= +github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085 h1:VbVPS7aWjDY56dJ0B92AQk1cftP/Q7QK5qZ8FWg6WzQ= +github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 56b3f7b15..166c1e859 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -56,14 +56,13 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { }) } - // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{1} + cycles := []int{4} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) t.Run(ns+"-fastretrieval-CAR", func(t *testing.T) { runTest(t, n, true, true) }) t.Run(ns+"-fastretrieval-NoCAR", func(t *testing.T) { runTest(t, n, true, false) }) - t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, true, false) }) + t.Run(ns+"-stdretrieval-CAR", func(t *testing.T) { runTest(t, n, false, true) }) t.Run(ns+"-stdretrieval-NoCAR", func(t *testing.T) { runTest(t, n, false, false) }) } } @@ -101,8 +100,7 @@ func TestDealCyclesConcurrent(t *testing.T) { }) } - // TODO: add 2, 4, 8, more when this graphsync issue is fixed: https://github.com/ipfs/go-graphsync/issues/175# - cycles := []int{2} + cycles := []int{2, 4, 8, 16} for _, n := range cycles { n := n ns := fmt.Sprintf("%d", n) @@ -133,7 +131,10 @@ func TestSimultanenousTransferLimit(t *testing.T) { // so that the deal starts sealing in time startEpoch := abi.ChainEpoch(2 << 12) - const graphsyncThrottle = 2 + const ( + graphsyncThrottle = 2 + concurrency = 20 + ) runTest := func(t *testing.T) { client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), kit.ConstructorOpts( node.ApplyIf(node.IsType(repo.StorageMiner), node.Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync(graphsyncThrottle))), @@ -174,11 +175,10 @@ func TestSimultanenousTransferLimit(t *testing.T) { } }() - const concurrency = 20 t.Logf("running concurrent deals: %d", concurrency) dh.RunConcurrentDeals(kit.RunConcurrentDealsOpts{ - N: concurrency, // TODO: set to 20 after https://github.com/ipfs/go-graphsync/issues/175 is fixed + N: concurrency, FastRetrieval: true, StartEpoch: startEpoch, }) From 5e66960ab6913e769d04c74b2215e0dab9933dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 20 Jul 2021 17:51:09 +0100 Subject: [PATCH 8/9] fix merge error. --- itests/deals_concurrent_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/itests/deals_concurrent_test.go b/itests/deals_concurrent_test.go index 81acec809..241c9071d 100644 --- a/itests/deals_concurrent_test.go +++ b/itests/deals_concurrent_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/stretchr/testify/require" datatransfer "github.com/filecoin-project/go-data-transfer" @@ -55,7 +56,7 @@ func TestDealWithMarketAndMinerNode(t *testing.T) { }) } - // this test is expensive because we don't use mock proofs; do a single cycle. + // this test is expensive because we don't use mock proofs; do a single cycle. cycles := []int{4} for _, n := range cycles { n := n @@ -100,7 +101,7 @@ func TestDealCyclesConcurrent(t *testing.T) { }) } - // this test is cheap because we use mock proofs, do various cycles + // this test is cheap because we use mock proofs, do various cycles cycles := []int{2, 4, 8, 16} for _, n := range cycles { n := n From 21f9dc8096bc6a6f5316217ef84977a5398351ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Thu, 22 Jul 2021 23:47:28 +0100 Subject: [PATCH 9/9] update graphsync to an actual release. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 97597629d..1725754b4 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-fs-lock v0.0.6 - github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085 + github.com/ipfs/go-graphsync v0.6.5 github.com/ipfs/go-ipfs-blockstore v1.0.3 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/go.sum b/go.sum index b43a2bd3d..ee34abe5c 100644 --- a/go.sum +++ b/go.sum @@ -628,8 +628,8 @@ github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CE github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0= github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY= github.com/ipfs/go-graphsync v0.6.4/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg= -github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085 h1:VbVPS7aWjDY56dJ0B92AQk1cftP/Q7QK5qZ8FWg6WzQ= -github.com/ipfs/go-graphsync v0.6.5-0.20210720015721-3e92e95a9085/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= +github.com/ipfs/go-graphsync v0.6.5 h1:YAJl6Yit23PQcaawzb1rPK9PSnbbq2jjMRPpRpJ0Y5U= +github.com/ipfs/go-graphsync v0.6.5/go.mod h1:GdHT8JeuIZ0R4lSjFR16Oe4zPi5dXwKi9zR9ADVlcdk= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=