From 99796220f928106d419f6e6cd8b7e94d980c772e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jul 2020 20:35:55 +0200 Subject: [PATCH] Support fast-retrieval deals --- api/api_storage.go | 6 ++++-- api/test/ccupgrade.go | 2 +- api/test/deals.go | 15 ++++++++------- api/test/mining.go | 2 +- go.mod | 6 +++--- go.sum | 12 ++++++------ markets/storageadapter/provider.go | 1 + node/node_test.go | 12 +++++++++--- node/repo/fsrepo.go | 2 +- node/repo/interface.go | 6 +++--- node/repo/memrepo.go | 4 ++-- 11 files changed, 39 insertions(+), 29 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index a53efe41a..6402de2b6 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -3,6 +3,10 @@ package api import ( "bytes" "context" + "time" + + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/lotus/chain/types" @@ -10,8 +14,6 @@ import ( "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/sector-storage/storiface" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/ipfs/go-cid" - "time" ) // StorageMiner is a low-level interface to the Filecoin network storage miner node diff --git a/api/test/ccupgrade.go b/api/test/ccupgrade.go index 78a03b4dd..69cf1fadf 100644 --- a/api/test/ccupgrade.go +++ b/api/test/ccupgrade.go @@ -78,7 +78,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal(err) } - makeDeal(t, ctx, 6, client, miner, false) + makeDeal(t, ctx, 6, client, miner, false, false) // Validate upgrade diff --git a/api/test/deals.go b/api/test/deals.go index 5eb7f726b..0a41741b3 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -37,7 +37,7 @@ func init() { build.InsecurePoStValidation = true } -func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) { +func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport, fastRet bool) { _ = os.Setenv("BELLMAN_NO_GPU", "1") ctx := context.Background() @@ -67,7 +67,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport } }() - makeDeal(t, ctx, 6, client, miner, carExport) + makeDeal(t, ctx, 6, client, miner, carExport, fastRet) atomic.AddInt64(&mine, -1) fmt.Println("shutting down mining") @@ -105,15 +105,15 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { } }() - makeDeal(t, ctx, 6, client, miner, false) - makeDeal(t, ctx, 7, client, miner, false) + makeDeal(t, ctx, 6, client, miner, false, false) + makeDeal(t, ctx, 7, client, miner, false, false) atomic.AddInt64(&mine, -1) fmt.Println("shutting down mining") <-done } -func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) { +func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport, fastRet bool) { data := make([]byte, 1600) rand.New(rand.NewSource(int64(rseed))).Read(data) @@ -125,7 +125,7 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod fmt.Println("FILE CID: ", fcid) - deal := startDeal(t, ctx, miner, client, fcid) + deal := startDeal(t, ctx, miner, client, fcid, fastRet) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) @@ -136,7 +136,7 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod testRetrieval(t, ctx, err, client, fcid, carExport, data) } -func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid { +func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid { maddr, err := miner.ActorAddress(ctx) if err != nil { t.Fatal(err) @@ -152,6 +152,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client Miner: maddr, EpochPrice: types.NewInt(1000000), MinBlocksDuration: 100, + FastRetrieval: fastRet, }) if err != nil { t.Fatalf("%+v", err) diff --git a/api/test/mining.go b/api/test/mining.go index 766021a69..d2b56758e 100644 --- a/api/test/mining.go +++ b/api/test/mining.go @@ -189,7 +189,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo } }() - deal := startDeal(t, ctx, provider, client, fcid) + deal := startDeal(t, ctx, provider, client, fcid, false) // TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this time.Sleep(time.Second) diff --git a/go.mod b/go.mod index 1d5e82c60..c6727b240 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 - github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38 + github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v0.4.0 @@ -28,10 +28,10 @@ require ( github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b - github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324 + github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e github.com/filecoin-project/specs-actors v0.7.1 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea - github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e + github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-kit/kit v0.10.0 github.com/go-ole/go-ole v1.2.4 // indirect diff --git a/go.sum b/go.sum index 231291385..12d1e3ff3 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,8 @@ github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mo github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= -github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38 h1:B2gUde2DlfCb5YMYNVems2orobxC3KhrX3migym1IOQ= -github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= @@ -256,8 +256,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/ github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM= github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246 h1:NfYQRmVRe0LzlNbK5Ket3vbBOwFD5TvtcNtfo/Sd8mg= github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY= -github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324 h1:MmxTkkhQMGWH3fr4BPpGoFQocG1dTvAAbkL3VEaZcsY= -github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324/go.mod h1:r12d7tsmJKz8QDGoCvl65Ay2al6mOgDqxAGUxbyrgMs= +github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e h1:wt7UpNjnCcCFGBZ9Hj3n+4WryepVFXHFaGUWmAVupoI= +github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= @@ -269,8 +269,8 @@ github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sg github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e h1:touwKs24SmZ1RCQzzW24jgbTHTKPKWF7evZTudQ4Z5g= -github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e/go.mod h1:SXO4VnXG056B/lXHL8HZv54eMqlsyynm+v93BlLwlOY= +github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c h1:F6guH363a+fpew1zkgoez4/U0RqW4ph6GVXR23lVwng= +github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c/go.mod h1:SXO4VnXG056B/lXHL8HZv54eMqlsyynm+v93BlLwlOY= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 32bed3c2f..4341d1740 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -95,6 +95,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch, EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch, }, + KeepUnsealed: deal.FastRetrieval, }) if err != nil { return xerrors.Errorf("AddPiece failed: %s", err) diff --git a/node/node_test.go b/node/node_test.go index 9316e7ef7..64d59fc50 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -459,10 +459,10 @@ func TestAPIDealFlow(t *testing.T) { logging.SetLogLevel("storageminer", "ERROR") t.Run("TestDealFlow", func(t *testing.T) { - test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false) + test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false, false) }) t.Run("WithExportedCAR", func(t *testing.T) { - test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, true) + test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, true, false) }) t.Run("TestDoubleDealFlow", func(t *testing.T) { test.TestDoubleDealFlow(t, mockSbBuilder, 10*time.Millisecond) @@ -480,7 +480,13 @@ func TestAPIDealFlowReal(t *testing.T) { logging.SetLogLevel("sub", "ERROR") logging.SetLogLevel("storageminer", "ERROR") - test.TestDealFlow(t, builder, time.Second, false) + t.Run("basic", func(t *testing.T) { + test.TestDealFlow(t, builder, time.Second, false, false) + }) + + t.Run("fast-retrieval", func(t *testing.T) { + test.TestDealFlow(t, builder, time.Second, false, true) + }) } func TestDealMining(t *testing.T) { diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 6342a918c..c1f0db9bb 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/filecoin-project/sector-storage/fsutil" "io" "io/ioutil" "os" @@ -21,6 +20,7 @@ import ( "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" + "github.com/filecoin-project/sector-storage/fsutil" "github.com/filecoin-project/sector-storage/stores" "github.com/filecoin-project/lotus/chain/types" diff --git a/node/repo/interface.go b/node/repo/interface.go index 5dc4c3c88..17336d413 100644 --- a/node/repo/interface.go +++ b/node/repo/interface.go @@ -2,13 +2,13 @@ package repo import ( "errors" - "github.com/filecoin-project/sector-storage/fsutil" - - "github.com/filecoin-project/sector-storage/stores" "github.com/ipfs/go-datastore" "github.com/multiformats/go-multiaddr" + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/lotus/chain/types" ) diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 818f34745..229705929 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -2,7 +2,6 @@ package repo import ( "encoding/json" - "github.com/filecoin-project/sector-storage/fsutil" "io/ioutil" "os" "path/filepath" @@ -15,6 +14,8 @@ import ( "github.com/multiformats/go-multiaddr" "golang.org/x/xerrors" + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/sector-storage/stores" @@ -82,7 +83,6 @@ func (lmem *lockedMemRepo) Stat(path string) (fsutil.FsStat, error) { return fsutil.Statfs(path) } - func (lmem *lockedMemRepo) DiskUsage(path string) (int64, error) { si, err := fsutil.FileSize(path) if err != nil {