cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into next

Browse Source
This commit is contained in:
Łukasz Magiera
2020-10-06 23:54:59 +02:00
parent 60ad339033 2bf42d5318
commit 2f70a91665
35 changed files with 1744 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/impl/backup.go
+67
View File
@@ -0,0 +1,67 @@
package impl
import (
"os"
"path/filepath"
"strings"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func backup(mds dtypes.MetadataDS, fpath string) error {
bb, ok := os.LookupEnv("LOTUS_BACKUP_BASE_PATH")
if !ok {
return xerrors.Errorf("LOTUS_BACKUP_BASE_PATH env var not set")
}
bds, ok := mds.(*backupds.Datastore)
if !ok {
return xerrors.Errorf("expected a backup datastore")
}
bb, err := homedir.Expand(bb)
if err != nil {
return xerrors.Errorf("expanding base path: %w", err)
}
bb, err = filepath.Abs(bb)
if err != nil {
return xerrors.Errorf("getting absolute base path: %w", err)
}
fpath, err = homedir.Expand(fpath)
if err != nil {
return xerrors.Errorf("expanding file path: %w", err)
}
fpath, err = filepath.Abs(fpath)
if err != nil {
return xerrors.Errorf("getting absolute file path: %w", err)
}
if !strings.HasPrefix(fpath, bb) {
return xerrors.Errorf("backup file name (%s) must be inside base path (%s)", fpath, bb)
}
out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return xerrors.Errorf("open %s: %w", fpath, err)
}
if err := bds.Backup(out); err != nil {
if cerr := out.Close(); cerr != nil {
log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err)
}
return xerrors.Errorf("backup error: %w", err)
}
if err := out.Close(); err != nil {
return xerrors.Errorf("closing backup file: %w", err)
}
return nil
}
node/impl/common/common.go
+6
View File
@@ -121,6 +121,12 @@ func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo,
func (a *CommonAPI) NetAutoNatStatus(ctx context.Context) (i api.NatInfo, err error) {
autonat := a.RawHost.(*basichost.BasicHost).AutoNat
if autonat == nil {
return api.NatInfo{
Reachability: network.ReachabilityUnknown,
}, nil
}
var maddr string
if autonat.Status() == network.ReachabilityPublic {
pa, err := autonat.PublicAddr()
node/impl/full.go
+9
View File
@@ -1,6 +1,8 @@
package impl
import (
"context"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/api"
@@ -9,6 +11,7 @@ import (
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/impl/market"
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
var log = logging.Logger("node")
@@ -26,6 +29,12 @@ type FullNodeAPI struct {
full.WalletAPI
full.SyncAPI
full.BeaconAPI
DS dtypes.MetadataDS
}
func (n *FullNodeAPI) CreateBackup(ctx context.Context, fpath string) error {
return backup(n.DS, fpath)
}
var _ api.FullNode = &FullNodeAPI{}
node/impl/full/chain.go
+18 -12
View File
@@ -509,15 +509,11 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo
r, w := io.Pipe()
out := make(chan []byte)
go func() {
defer w.Close() //nolint:errcheck // it is a pipe
bw := bufio.NewWriterSize(w, 1<<20)
defer bw.Flush() //nolint:errcheck // it is a write to a pipe
if err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw); err != nil {
log.Errorf("chain export call failed: %s", err)
return
}
err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw)
bw.Flush() //nolint:errcheck // it is a write to a pipe
w.CloseWithError(err) //nolint:errcheck // it is a pipe
}()
go func() {
@@ -529,13 +525,23 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipo
log.Errorf("chain export pipe read failed: %s", err)
return
}
select {
case out <- buf[:n]:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
if n > 0 {
select {
case out <- buf[:n]:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
}
if err == io.EOF {
// send empty slice to indicate correct eof
select {
case out <- []byte{}:
case <-ctx.Done():
log.Warnf("export writer failed: %s", ctx.Err())
return
}
return
}
}
node/impl/storminer.go
+8 -2
View File
@@ -8,18 +8,18 @@ import (
"strconv"
"time"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/big"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/host"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-fil-markets/piecestore"
retrievalmarket "github.com/filecoin-project/go-fil-markets/retrievalmarket"
storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -56,6 +56,8 @@ type StorageMinerAPI struct {
DataTransfer dtypes.ProviderDataTransfer
Host host.Host
DS dtypes.MetadataDS
ConsiderOnlineStorageDealsConfigFunc dtypes.ConsiderOnlineStorageDealsConfigFunc
SetConsiderOnlineStorageDealsConfigFunc dtypes.SetConsiderOnlineStorageDealsConfigFunc
ConsiderOnlineRetrievalDealsConfigFunc dtypes.ConsiderOnlineRetrievalDealsConfigFunc
@@ -516,4 +518,8 @@ func (sm *StorageMinerAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.
return &ci, nil
}
func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error {
return backup(sm.DS, fpath)
}
var _ api.StorageMiner = &StorageMinerAPI{}
node/modules/lp2p/pubsub.go
+2 -2
View File
@@ -223,8 +223,8 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
},
AppSpecificWeight: 1,
// This sets the IP colocation threshold to 1 peer per
IPColocationFactorThreshold: 1,
// This sets the IP colocation threshold to 5 peers before we apply penalties
IPColocationFactorThreshold: 5,
IPColocationFactorWeight: -100,
// TODO we want to whitelist IPv6 /64s that belong to datacenters etc
// IPColocationFactorWhitelist: map[string]struct{}{},
node/modules/storage.go
+7 -1
View File
@@ -6,6 +6,7 @@ import (
"go.uber.org/fx"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/backupds"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
)
@@ -27,5 +28,10 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
}
func Datastore(r repo.LockedRepo) (dtypes.MetadataDS, error) {
return r.Datastore("/metadata")
mds, err := r.Datastore("/metadata")
if err != nil {
return nil, err
}
return backupds.Wrap(mds), nil
}
node/modules/storageminer.go
+8
View File
@@ -44,6 +44,7 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-storedcounter"
"github.com/filecoin-project/specs-actors/actors/builtin"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -468,6 +469,13 @@ func BasicDealFilter(user dtypes.DealFilter) func(onlineOk dtypes.ConsiderOnline
return false, fmt.Sprintf("cannot seal a sector before %s", deal.Proposal.StartEpoch), nil
}
// Reject if it's more than 7 days in the future
// TODO: read from cfg
maxStartEpoch := ht + abi.ChainEpoch(7*builtin.EpochsInDay)
if deal.Proposal.StartEpoch > maxStartEpoch {
return false, fmt.Sprintf("deal start epoch is too far in the future: %s > %s", deal.Proposal.StartEpoch, maxStartEpoch), nil
}
if user != nil {
return user(ctx, deal)
}
node/repo/fsrepo.go
+12
View File
@@ -226,11 +226,23 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) {
}, nil
}
// Like Lock, except datastores will work in read-only mode
func (fsr *FsRepo) LockRO(repoType RepoType) (LockedRepo, error) {
lr, err := fsr.Lock(repoType)
if err != nil {
return nil, err
}
lr.(*fsLockedRepo).readonly = true
return lr, nil
}
type fsLockedRepo struct {
path string
configPath string
repoType RepoType
closer io.Closer
readonly bool
ds map[string]datastore.Batching
dsErr error
node/repo/fsrepo_ds.go
+11 -7
View File
@@ -14,7 +14,7 @@ import (
ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
)
type dsCtor func(path string) (datastore.Batching, error)
type dsCtor func(path string, readonly bool) (datastore.Batching, error)
var fsDatastores = map[string]dsCtor{
"chain": chainBadgerDs,
@@ -26,9 +26,10 @@ var fsDatastores = map[string]dsCtor{
"client": badgerDs, // client specific
}
func chainBadgerDs(path string) (datastore.Batching, error) {
func chainBadgerDs(path string, readonly bool) (datastore.Batching, error) {
opts := badger.DefaultOptions
opts.GcInterval = 0 // disable GC for chain datastore
opts.ReadOnly = readonly
opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).
WithValueThreshold(1 << 10)
@@ -36,23 +37,26 @@ func chainBadgerDs(path string) (datastore.Batching, error) {
return badger.NewDatastore(path, &opts)
}
func badgerDs(path string) (datastore.Batching, error) {
func badgerDs(path string, readonly bool) (datastore.Batching, error) {
opts := badger.DefaultOptions
opts.ReadOnly = readonly
opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).
WithValueThreshold(1 << 10)
return badger.NewDatastore(path, &opts)
}
func levelDs(path string) (datastore.Batching, error) {
func levelDs(path string, readonly bool) (datastore.Batching, error) {
return levelds.NewDatastore(path, &levelds.Options{
Compression: ldbopts.NoCompression,
NoSync: false,
Strict: ldbopts.StrictAll,
ReadOnly: readonly,
})
}
func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error) {
func (fsr *fsLockedRepo) openDatastores(readonly bool) (map[string]datastore.Batching, error) {
if err := os.MkdirAll(fsr.join(fsDatastore), 0755); err != nil {
return nil, xerrors.Errorf("mkdir %s: %w", fsr.join(fsDatastore), err)
}
@@ -63,7 +67,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error)
prefix := datastore.NewKey(p)
// TODO: optimization: don't init datastores we don't need
ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)))
ds, err := ctor(fsr.join(filepath.Join(fsDatastore, p)), readonly)
if err != nil {
return nil, xerrors.Errorf("opening datastore %s: %w", prefix, err)
}
@@ -78,7 +82,7 @@ func (fsr *fsLockedRepo) openDatastores() (map[string]datastore.Batching, error)
func (fsr *fsLockedRepo) Datastore(ns string) (datastore.Batching, error) {
fsr.dsOnce.Do(func() {
fsr.ds, fsr.dsErr = fsr.openDatastores()
fsr.ds, fsr.dsErr = fsr.openDatastores(fsr.readonly)
})
if fsr.dsErr != nil {