From 43f7fd5e10557f1338f68bde107a0d868e686648 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 1 Oct 2021 18:26:38 +1000 Subject: [PATCH] traversals: limit maximum number of DAG links to traverse Impacts CommP and graphsync transfers --- node/config/def.go | 12 ++++++++++++ node/modules/client.go | 3 ++- node/modules/graphsync.go | 10 +++++++++- node/modules/storageminer.go | 10 +++++++++- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index f5117b56f..8dafbbfb2 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -2,6 +2,8 @@ package config import ( "encoding" + "os" + "strconv" "time" "github.com/ipfs/go-cid" @@ -24,6 +26,16 @@ const ( RetrievalPricingExternalMode = "external" ) +// MaxTraversalLinks configures the maximum number of links to traverse in a DAG while calculating +// CommP and traversing a DAG with graphsync; invokes a budget on DAG depth and density. +var MaxTraversalLinks uint64 = 32 * (1 << 20) + +func init() { + if envMaxTraversal, err := strconv.ParseUint(os.Getenv("LOTUS_MAX_TRAVERSAL_LINKS"), 10, 64); err == nil { + MaxTraversalLinks = envMaxTraversal + } +} + func (b *BatchFeeConfig) FeeForSectors(nSectors int) abi.TokenAmount { return big.Add(big.Int(b.Base), big.Mul(big.NewInt(int64(nSectors)), big.Int(b.PerSector))) } diff --git a/node/modules/client.go b/node/modules/client.go index 0a1c1039b..4d988d98a 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -36,6 +36,7 @@ import ( "github.com/filecoin-project/lotus/markets" marketevents "github.com/filecoin-project/lotus/markets/loggers" "github.com/filecoin-project/lotus/markets/retrievaladapter" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/impl/full" payapi "github.com/filecoin-project/lotus/node/impl/paych" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -182,7 +183,7 @@ func StorageClient(lc fx.Lifecycle, h host.Host, dataTransfer dtypes.ClientDataT marketsRetryParams := smnet.RetryParameters(time.Second, 5*time.Minute, 15, 5) net := smnet.NewFromLibp2pHost(h, marketsRetryParams) - c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second)) + c, err := storageimpl.NewClient(net, dataTransfer, discovery, deals, scn, accessor, storageimpl.DealPollingInterval(time.Second), storageimpl.MaxTraversalLinks(config.MaxTraversalLinks)) if err != nil { return nil, err } diff --git a/node/modules/graphsync.go b/node/modules/graphsync.go index 8fa8368ca..839508900 100644 --- a/node/modules/graphsync.go +++ b/node/modules/graphsync.go @@ -9,6 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "go.uber.org/fx" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -20,7 +21,14 @@ func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForRetrieval graphsyncNetwork := gsnet.NewFromLibp2pHost(h) lsys := storeutil.LinkSystemForBlockstore(clientBs) - gs := graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, lsys, graphsyncimpl.RejectAllRequestsByDefault(), graphsyncimpl.MaxInProgressIncomingRequests(parallelTransfersForStorage), graphsyncimpl.MaxInProgressOutgoingRequests(parallelTransfersForRetrieval)) + gs := graphsyncimpl.New(helpers.LifecycleCtx(mctx, lc), + graphsyncNetwork, + lsys, + graphsyncimpl.RejectAllRequestsByDefault(), + graphsyncimpl.MaxInProgressIncomingRequests(parallelTransfersForStorage), + graphsyncimpl.MaxInProgressOutgoingRequests(parallelTransfersForRetrieval), + graphsyncimpl.MaxLinksPerIncomingRequests(config.MaxTraversalLinks), + graphsyncimpl.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks)) chainLinkSystem := storeutil.LinkSystemForBlockstore(chainBs) err := gs.RegisterPersistenceOption("chainstore", chainLinkSystem) if err != nil { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 383e6879d..0f74a8d58 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -38,6 +38,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" graphsync "github.com/ipfs/go-graphsync/impl" + graphsyncimpl "github.com/ipfs/go-graphsync/impl" gsnet "github.com/ipfs/go-graphsync/network" "github.com/ipfs/go-graphsync/storeutil" "github.com/libp2p/go-libp2p-core/host" @@ -398,7 +399,14 @@ func StagingGraphsync(parallelTransfersForStorage uint64, parallelTransfersForRe return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.StagingBlockstore, h host.Host) dtypes.StagingGraphsync { graphsyncNetwork := gsnet.NewFromLibp2pHost(h) lsys := storeutil.LinkSystemForBlockstore(ibs) - gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, lsys, graphsync.RejectAllRequestsByDefault(), graphsync.MaxInProgressOutgoingRequests(parallelTransfersForStorage), graphsync.MaxInProgressIncomingRequests(parallelTransfersForRetrieval)) + gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), + graphsyncNetwork, + lsys, + graphsync.RejectAllRequestsByDefault(), + graphsync.MaxInProgressIncomingRequests(parallelTransfersForRetrieval), + graphsync.MaxInProgressOutgoingRequests(parallelTransfersForStorage), + graphsyncimpl.MaxLinksPerIncomingRequests(config.MaxTraversalLinks), + graphsyncimpl.MaxLinksPerOutgoingRequests(config.MaxTraversalLinks)) return gs }