cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Support fast-retrieval deals

Browse Source
This commit is contained in:
Łukasz Magiera 2020-07-08 20:35:55 +02:00
parent 8e0d33a1fd
commit 99796220f9
11 changed files with 39 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/api_storage.go
View File

@ -3,6 +3,10 @@ package api
import (
"bytes"
"context"
"time"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/types"
@ -10,8 +14,6 @@ import (
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/sector-storage/storiface"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/ipfs/go-cid"
"time"
)
// StorageMiner is a low-level interface to the Filecoin network storage miner node

2
api/test/ccupgrade.go
View File

@ -78,7 +78,7 @@ func TestCCUpgrade(t *testing.T, b APIBuilder, blocktime time.Duration) {
t.Fatal(err)
}
makeDeal(t, ctx, 6, client, miner, false)
makeDeal(t, ctx, 6, client, miner, false, false)
// Validate upgrade

15
api/test/deals.go
View File

@ -37,7 +37,7 @@ func init() {
build.InsecurePoStValidation = true
}
func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport bool) {
func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport, fastRet bool) {
_ = os.Setenv("BELLMAN_NO_GPU", "1")
ctx := context.Background()
@ -67,7 +67,7 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration, carExport
}
}()
makeDeal(t, ctx, 6, client, miner, carExport)
makeDeal(t, ctx, 6, client, miner, carExport, fastRet)
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
@ -105,15 +105,15 @@ func TestDoubleDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
}
}()
makeDeal(t, ctx, 6, client, miner, false)
makeDeal(t, ctx, 7, client, miner, false)
makeDeal(t, ctx, 6, client, miner, false, false)
makeDeal(t, ctx, 7, client, miner, false, false)
atomic.AddInt64(&mine, -1)
fmt.Println("shutting down mining")
<-done
}
func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport bool) {
func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNodeAPI, miner TestStorageNode, carExport, fastRet bool) {
data := make([]byte, 1600)
rand.New(rand.NewSource(int64(rseed))).Read(data)
@ -125,7 +125,7 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
fmt.Println("FILE CID: ", fcid)
deal := startDeal(t, ctx, miner, client, fcid)
deal := startDeal(t, ctx, miner, client, fcid, fastRet)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
@ -136,7 +136,7 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
testRetrieval(t, ctx, err, client, fcid, carExport, data)
}
func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid) *cid.Cid {
func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client *impl.FullNodeAPI, fcid cid.Cid, fastRet bool) *cid.Cid {
maddr, err := miner.ActorAddress(ctx)
if err != nil {
t.Fatal(err)
@ -152,6 +152,7 @@ func startDeal(t *testing.T, ctx context.Context, miner TestStorageNode, client
Miner: maddr,
EpochPrice: types.NewInt(1000000),
MinBlocksDuration: 100,
FastRetrieval: fastRet,
})
if err != nil {
t.Fatalf("%+v", err)

2
api/test/mining.go
View File

@ -189,7 +189,7 @@ func TestDealMining(t *testing.T, b APIBuilder, blocktime time.Duration, carExpo
}
}()
deal := startDeal(t, ctx, provider, client, fcid)
deal := startDeal(t, ctx, provider, client, fcid, false)
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)

6
go.mod
View File

@ -18,7 +18,7 @@ require (
github.com/filecoin-project/filecoin-ffi v0.26.1-0.20200508175440-05b30afeb00d
github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38
github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-data-transfer v0.4.0
@ -28,10 +28,10 @@ require (
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
github.com/filecoin-project/go-statestore v0.1.0
github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324
github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e
github.com/filecoin-project/specs-actors v0.7.1
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e
github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-kit/kit v0.10.0
github.com/go-ole/go-ole v1.2.4 // indirect

12
go.sum
View File

@ -225,8 +225,8 @@ github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mo
github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38 h1:B2gUde2DlfCb5YMYNVems2orobxC3KhrX3migym1IOQ=
github.com/filecoin-project/go-bitfield v0.0.2-0.20200629135455-587b27927d38/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw=
github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@ -256,8 +256,8 @@ github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/
github.com/filecoin-project/sector-storage v0.0.0-20200615154852-728a47ab99d6/go.mod h1:M59QnAeA/oV+Z8oHFLoNpGMv0LZ8Rll+vHVXX7GirPM=
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246 h1:NfYQRmVRe0LzlNbK5Ket3vbBOwFD5TvtcNtfo/Sd8mg=
github.com/filecoin-project/sector-storage v0.0.0-20200625154333-98ef8e4ef246/go.mod h1:8f0hWDzzIi1hKs4IVKH9RnDsO4LEHVz8BNat0okDOuY=
github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324 h1:MmxTkkhQMGWH3fr4BPpGoFQocG1dTvAAbkL3VEaZcsY=
github.com/filecoin-project/sector-storage v0.0.0-20200701092105-a2de752a3324/go.mod h1:r12d7tsmJKz8QDGoCvl65Ay2al6mOgDqxAGUxbyrgMs=
github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e h1:wt7UpNjnCcCFGBZ9Hj3n+4WryepVFXHFaGUWmAVupoI=
github.com/filecoin-project/sector-storage v0.0.0-20200708175126-9af64c9b217e/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA=
github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
@ -269,8 +269,8 @@ github.com/filecoin-project/specs-storage v0.1.0 h1:PkDgTOT5W5Ao7752onjDl4QSv+sg
github.com/filecoin-project/specs-storage v0.1.0/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=
github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e h1:touwKs24SmZ1RCQzzW24jgbTHTKPKWF7evZTudQ4Z5g=
github.com/filecoin-project/storage-fsm v0.0.0-20200707191927-b838ed4e859e/go.mod h1:SXO4VnXG056B/lXHL8HZv54eMqlsyynm+v93BlLwlOY=
github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c h1:F6guH363a+fpew1zkgoez4/U0RqW4ph6GVXR23lVwng=
github.com/filecoin-project/storage-fsm v0.0.0-20200707194229-bc5e298e2b4c/go.mod h1:SXO4VnXG056B/lXHL8HZv54eMqlsyynm+v93BlLwlOY=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=

1
markets/storageadapter/provider.go
View File

@ -95,6 +95,7 @@ func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagema
StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch,
EndEpoch: deal.ClientDealProposal.Proposal.EndEpoch,
},
KeepUnsealed: deal.FastRetrieval,
})
if err != nil {
return xerrors.Errorf("AddPiece failed: %s", err)

12
node/node_test.go
View File

@ -459,10 +459,10 @@ func TestAPIDealFlow(t *testing.T) {
logging.SetLogLevel("storageminer", "ERROR")
t.Run("TestDealFlow", func(t *testing.T) {
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false)
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, false, false)
})
t.Run("WithExportedCAR", func(t *testing.T) {
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, true)
test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond, true, false)
})
t.Run("TestDoubleDealFlow", func(t *testing.T) {
test.TestDoubleDealFlow(t, mockSbBuilder, 10*time.Millisecond)
@ -480,7 +480,13 @@ func TestAPIDealFlowReal(t *testing.T) {
logging.SetLogLevel("sub", "ERROR")
logging.SetLogLevel("storageminer", "ERROR")
test.TestDealFlow(t, builder, time.Second, false)
t.Run("basic", func(t *testing.T) {
test.TestDealFlow(t, builder, time.Second, false, false)
})
t.Run("fast-retrieval", func(t *testing.T) {
test.TestDealFlow(t, builder, time.Second, false, true)
})
}
func TestDealMining(t *testing.T) {

2
node/repo/fsrepo.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/filecoin-project/sector-storage/fsutil"
"io"
"io/ioutil"
"os"
@ -21,6 +20,7 @@ import (
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/chain/types"

6
node/repo/interface.go
View File

@ -2,13 +2,13 @@ package repo
import (
"errors"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/sector-storage/stores"
"github.com/ipfs/go-datastore"
"github.com/multiformats/go-multiaddr"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/sector-storage/stores"
"github.com/filecoin-project/lotus/chain/types"
)

4
node/repo/memrepo.go
View File

@ -2,7 +2,6 @@ package repo
import (
"encoding/json"
"github.com/filecoin-project/sector-storage/fsutil"
"io/ioutil"
"os"
"path/filepath"
@ -15,6 +14,8 @@ import (
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"github.com/filecoin-project/sector-storage/fsutil"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/sector-storage/stores"
@ -82,7 +83,6 @@ func (lmem *lockedMemRepo) Stat(path string) (fsutil.FsStat, error) {
return fsutil.Statfs(path)
}
func (lmem *lockedMemRepo) DiskUsage(path string) (int64, error) {
si, err := fsutil.FileSize(path)
if err != nil {