cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'feat/testnet2' into feat/event-states

Browse Source
This commit is contained in:
Łukasz Magiera 2020-01-13 15:52:43 +01:00
commit 34abc3d023
144 changed files with 2298 additions and 7110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.circleci/config.yml
View File

@ -37,14 +37,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}'
- 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-'
- 'v20-1k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params --proving-params 1024
- save_cache:
name: Save parameters cache
key: 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}'
key: 'v20-1k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
@ -101,7 +100,7 @@ jobs:
description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
coverage:
type: string
default: -coverprofile=coverage.txt
default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
description: Coverage flag. Set to the empty string to disable.
codecov-upload:
type: boolean
@ -191,7 +190,7 @@ jobs:
- install-deps
- go/mod-download
- run:
command: make buildall
command: make build
no_output_timeout: 30m
- store_artifacts:
path: lotus
@ -254,7 +253,8 @@ workflows:
jobs:
- lint-changes:
args: "--new-from-rev origin/master"
- test
- test:
codecov-upload: true
- test-short:
go-test-flags: "--timeout 10m --short"
- mod-tidy-check

2
.codecov.yml
View File

@ -1 +1,3 @@
comment: off
ignore:
- "cbor_gen.go"

5
api/api_full.go
View File

@ -36,15 +36,17 @@ type FullNode interface {
ChainGetGenesis(context.Context) (*types.TipSet, error)
ChainTipSetWeight(context.Context, *types.TipSet) (types.BigInt, error)
ChainGetNode(ctx context.Context, p string) (interface{}, error)
ChainGetMessage(context.Context, cid.Cid) (*types.Message, error)
// syncer
SyncState(context.Context) (*SyncState, error)
SyncSubmitBlock(ctx context.Context, blk *types.BlockMsg) error
SyncIncomingBlocks(ctx context.Context) (<-chan *types.BlockHeader, error)
SyncMarkBad(ctx context.Context, bcid cid.Cid) error
// messages
MpoolPending(context.Context, *types.TipSet) ([]*types.SignedMessage, error)
MpoolPush(context.Context, *types.SignedMessage) error // TODO: remove
MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error)
MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error) // get nonce, sign, push
MpoolGetNonce(context.Context, address.Address) (uint64, error)
MpoolSub(context.Context) (<-chan MpoolUpdate, error)
@ -95,6 +97,7 @@ type FullNode interface {
StateReplay(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error)
StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error)
StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error)
StateListMessages(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error)
StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)

2
api/api_storage.go
View File

@ -4,7 +4,7 @@ import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/go-sectorbuilder"
)
// alias because cbor-gen doesn't like non-alias types

21
api/apistruct/struct.go
View File

@ -3,7 +3,7 @@ package apistruct
import (
"context"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/network"
@ -54,13 +54,15 @@ type FullNodeStruct struct {
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"`
ChainGetNode func(ctx context.Context, p string) (interface{}, error) `perm:"read"`
ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"`
SyncState func(context.Context) (*api.SyncState, error) `perm:"read"`
SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"`
SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"`
SyncMarkBad func(ctx context.Context, bcid cid.Cid) error `perm:"admin"`
MpoolPending func(context.Context, *types.TipSet) ([]*types.SignedMessage, error) `perm:"read"`
MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"`
MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"`
MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"`
MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"`
@ -111,6 +113,7 @@ type FullNodeStruct struct {
StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"`
StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
StateMinerSectorCount func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error) `perm:"read"`
StateListMessages func(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) `perm:"read"`
MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
@ -232,7 +235,7 @@ func (c *FullNodeStruct) MpoolPending(ctx context.Context, ts *types.TipSet) ([]
return c.Internal.MpoolPending(ctx, ts)
}
func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) error {
func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) {
return c.Internal.MpoolPush(ctx, smsg)
}
@ -348,6 +351,10 @@ func (c *FullNodeStruct) ChainGetNode(ctx context.Context, p string) (interface{
return c.Internal.ChainGetNode(ctx, p)
}
func (c *FullNodeStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) {
return c.Internal.ChainGetMessage(ctx, mc)
}
func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
return c.Internal.SyncState(ctx)
}
@ -360,6 +367,10 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.
return c.Internal.SyncIncomingBlocks(ctx)
}
func (c *FullNodeStruct) SyncMarkBad(ctx context.Context, bcid cid.Cid) error {
return c.Internal.SyncMarkBad(ctx, bcid)
}
func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
return c.Internal.StateMinerSectors(ctx, addr, ts)
}
@ -447,6 +458,10 @@ func (c *FullNodeStruct) StateGetReceipt(ctx context.Context, msg cid.Cid, ts *t
return c.Internal.StateGetReceipt(ctx, msg, ts)
}
func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) {
return c.Internal.StateListMessages(ctx, match, ts, toht)
}
func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error {
return c.Internal.MarketEnsureAvailable(ctx, addr, amt)
}

2
api/test/deals.go
View File

@ -11,7 +11,7 @@ import (
"testing"
"time"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"

12
build/genesis.go
View File

@ -1,16 +1,22 @@
package build
import rice "github.com/GeertJohan/go.rice"
import (
rice "github.com/GeertJohan/go.rice"
logging "github.com/ipfs/go-log/v2"
)
// moved from now-defunct build/paramfetch.go
var log = logging.Logger("build")
func MaybeGenesis() []byte {
builtinGen, err := rice.FindBox("genesis")
if err != nil {
log.Warn("loading built-in genesis: %s", err)
log.Warnf("loading built-in genesis: %s", err)
return nil
}
genBytes, err := builtinGen.Bytes("devnet.car")
if err != nil {
log.Warn("loading built-in genesis: %s", err)
log.Warnf("loading built-in genesis: %s", err)
}
return genBytes

5
build/parameters.go Normal file
View File

@ -0,0 +1,5 @@
package build
import rice "github.com/GeertJohan/go.rice"
var ParametersJson = rice.MustFindBox("proof-params").MustBytes("parameters.json")

189
build/paramfetch.go
View File

@ -1,189 +0,0 @@
package build
import (
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
rice "github.com/GeertJohan/go.rice"
logging "github.com/ipfs/go-log"
"github.com/minio/blake2b-simd"
"go.uber.org/multierr"
"golang.org/x/xerrors"
pb "gopkg.in/cheggaaa/pb.v1"
)
var log = logging.Logger("build")
//const gateway = "http://198.211.99.118/ipfs/"
const gateway = "https://ipfs.io/ipfs/"
const paramdir = "/var/tmp/filecoin-proof-parameters"
const dirEnv = "FIL_PROOFS_PARAMETER_CACHE"
type paramFile struct {
Cid string `json:"cid"`
Digest string `json:"digest"`
SectorSize uint64 `json:"sector_size"`
}
type fetch struct {
wg sync.WaitGroup
fetchLk sync.Mutex
errs []error
}
func getParamDir() string {
if os.Getenv(dirEnv) == "" {
return paramdir
}
return os.Getenv(dirEnv)
}
func GetParams(storageSize uint64) error {
if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) {
return err
}
var params map[string]paramFile
paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json")
if err := json.Unmarshal(paramBytes, &params); err != nil {
return err
}
ft := &fetch{}
for name, info := range params {
if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") {
continue
}
ft.maybeFetchAsync(name, info)
}
return ft.wait()
}
func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
ft.wg.Add(1)
go func() {
defer ft.wg.Done()
path := filepath.Join(getParamDir(), name)
err := ft.checkFile(path, info)
if !os.IsNotExist(err) && err != nil {
log.Warn(err)
}
if err == nil {
return
}
ft.fetchLk.Lock()
defer ft.fetchLk.Unlock()
if err := doFetch(path, info); err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err))
return
}
err = ft.checkFile(path, info)
if err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err))
err := os.Remove(path)
if err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err))
}
}
}()
}
func (ft *fetch) checkFile(path string, info paramFile) error {
if os.Getenv("TRUST_PARAMS") == "1" {
log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
return nil
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
h := blake2b.New512()
if _, err := io.Copy(h, f); err != nil {
return err
}
sum := h.Sum(nil)
strSum := hex.EncodeToString(sum[:16])
if strSum == info.Digest {
log.Infof("Parameter file %s is ok", path)
return nil
}
return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
}
func (ft *fetch) wait() error {
ft.wg.Wait()
return multierr.Combine(ft.errs...)
}
func doFetch(out string, info paramFile) error {
gw := os.Getenv("IPFS_GATEWAY")
if gw == "" {
gw = gateway
}
log.Infof("Fetching %s from %s", out, gw)
outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return err
}
defer outf.Close()
fStat, err := outf.Stat()
if err != nil {
return err
}
header := http.Header{}
header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-")
url, err := url.Parse(gw + info.Cid)
if err != nil {
return err
}
log.Infof("GET %s", url)
req := http.Request{
Method: "GET",
URL: url,
Header: header,
Close: true,
}
resp, err := http.DefaultClient.Do(&req)
if err != nil {
return err
}
defer resp.Body.Close()
bar := pb.New64(resp.ContentLength)
bar.Units = pb.U_BYTES
bar.ShowSpeed = true
bar.Start()
_, err = io.Copy(outf, bar.NewProxyReader(resp.Body))
bar.Finish()
return err
}

11
build/params_shared.go
View File

@ -56,10 +56,8 @@ const SealRandomnessLookback = Finality
// Epochs
const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
// 1 / n
const SectorChallengeRatioDiv = 25
const MaxFallbackPostChallengeCount = 10
// Maximum lookback that randomness can be sourced from for a seal proof submission
const MaxSealLookback = SealRandomnessLookbackLimit + 2000
// /////
// Mining
@ -102,3 +100,8 @@ const BadBlockCacheSize = 1 << 15
// assuming 4000 messages per round, this lets us not lose any messages across a
// 10 block reorg.
const BlsSignatureCacheSize = 40000
// ///////
// Limits
const BlockMessageLimit = 512

20
build/proof-params/mkparamfetch.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
function b64() {
f="$1"
case `uname` in
Darwin)
base64 -i "$f"
;;
Linux)
base64 "$f" -w 0
;;
esac
printf "unsupported system" 1>&2
exit 1
}
sed "s/{{PARAMSJSON}}/$(b64 build/proof-params/parameters.json)/g" build/proof-params/paramfetch.sh.template > ./build/paramfetch.sh
chmod +x ./build/paramfetch.sh

95
build/proof-params/paramfetch.sh.template
View File

@ -1,95 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
PARAMS='{{PARAMSJSON}}'
die() {
echo "$@" >&2
exit 1
}
have_binary() {
type "$1" > /dev/null 2> /dev/null
}
check_writable() {
printf "" > "$1" && rm "$1"
}
try_download() {
url="$1"
output="$2"
command="$3"
util_name="$(set -- $command; echo "$1")"
if ! have_binary "$util_name"; then
return 1
fi
printf '==> Using %s to download "%s" to "%s"\n' "$util_name" "$url" "$output"
if eval "$command"; then
echo "==> Download complete!"
return
else
echo "error: couldn't download with $util_name ($?)"
return 1
fi
}
download() {
dl_url="$1"
dl_output="$2"
test "$#" -eq "2" || die "download requires exactly two arguments, was given $@"
if ! check_writable "$dl_output"; then
die "download error: cannot write to $dl_output"
fi
try_download "$dl_url" "$dl_output" "wget '$dl_url' -O '$dl_output'" && return
try_download "$dl_url" "$dl_output" "curl --silent --fail --output '$dl_output' '$dl_url'" && return
try_download "$dl_url" "$dl_output" "fetch '$dl_url' -o '$dl_output'" && return
try_download "$dl_url" "$dl_output" "http '$dl_url' > '$dl_output'" && return
try_download "$dl_url" "$dl_output" "ftp -o '$dl_output' '$dl_url'" && return
die "Unable to download $dl_url. exiting."
}
fetch_ipget() {
local dest="$1"
local cid="$2"
IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i"
./bin/ipget $IPGET_PARAMS -o "$dest" "$cid"
}
fetch_gateway() {
local dest="$1"
local cid="$2"
local url="http://198.211.99.118/ipfs/$cid"
download "$url" "$dest"
}
OUT_DIR="/var/tmp/filecoin-proof-parameters"
mkdir -p $OUT_DIR
printf $PARAMS | base64 -d | jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output | \
while read -r dest cid digest; do
if [[ -f "$dest" ]]; then
b2=$(b2sum "$dest" | head -c 32)
if [[ "$digest" == "$b2" ]]; then
echo "$dest exists and has correct hash"
continue
else
echo "$dest has incorrect hash"
rm -f "$dest"
fi
fi
echo "downloading $dest"
fetch_gateway "$dest" "$cid"
done

2
build/version.go
View File

@ -31,7 +31,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool {
}
// APIVersion is a semver version of the rpc api exposed
var APIVersion Version = newVer(0, 1, 5)
var APIVersion Version = newVer(0, 1, 6)
const (
majorMask = 0xff0000

2
chain/actors/actor_init.go
View File

@ -15,7 +15,7 @@ import (
"github.com/ipfs/go-cid"
"github.com/ipfs/go-hamt-ipld"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
mh "github.com/multiformats/go-multihash"
)

84
chain/actors/actor_miner.go
View File

@ -9,10 +9,10 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/go-amt-ipld"
"github.com/ipfs/go-cid"
@ -321,6 +321,14 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
// TODO: ensure normalization to ID address
maddr := vmctx.Message().To
if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback {
return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch)
}
if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback {
return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch)
}
ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
if err != nil {
return nil, aerrors.Wrap(err, "failed to get ticket randomness")
@ -467,14 +475,21 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
}
faults, nerr := self.FaultSet.AllMap()
ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
if lerr != nil {
return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
}
faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
if nerr != nil {
return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
}
activeFaults := uint64(0)
var sectorInfos []ffi.PublicSectorInfo
if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
if faults[id] {
activeFaults++
return nil
}
@ -512,7 +527,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
}
if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, len(faults)); !ok || lerr != nil {
sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil {
if lerr != nil {
// TODO: study PoST errors
return nil, aerrors.Absorb(lerr, 4, "PoST error")
@ -523,7 +538,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
}
// Post submission is successful!
if err := onSuccessfulPoSt(self, vmctx); err != nil {
if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil {
return nil, err
}
@ -834,6 +849,20 @@ func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMConte
return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
}
ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
if nerr != nil {
return nil, aerrors.HandleExternalError(nerr, "failed to load sector set")
}
cf, nerr := nfaults.Count()
if nerr != nil {
return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+")
}
if cf > 2*ss.Count {
return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count)
}
self.FaultSet = nfaults
self.LastFaultSubmission = vmctx.BlockHeight()
@ -909,7 +938,41 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
return nil, aerrors.New(1, "slashed miners can't perform election PoSt")
}
if err := onSuccessfulPoSt(self, vmctx); err != nil {
pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
if nerr != nil {
return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
}
ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
if nerr != nil {
return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
}
faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
if nerr != nil {
return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
}
activeFaults := uint64(0)
for f := range faults {
if f > amt.MaxIndex {
continue
}
var comms [][]byte
err := pss.Get(f, &comms)
if err != nil {
var notfound *amt.ErrNotFound
if !xerrors.As(err, &notfound) {
return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set")
}
continue
}
activeFaults++
}
if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO
return nil, err
}
@ -924,7 +987,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
return nil, nil
}
func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError {
// TODO: some sector upkeep stuff that is very haphazard and unclear in the spec
var mi MinerInfo
@ -937,7 +1000,12 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro
return aerrors.HandleExternalError(nerr, "failed to load proving set")
}
faults, nerr := self.FaultSet.All()
ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
if nerr != nil {
return aerrors.HandleExternalError(nerr, "failed to load sector set")
}
faults, nerr := self.FaultSet.All(2 * ss.Count)
if nerr != nil {
return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
}
@ -945,7 +1013,7 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro
self.FaultSet = types.NewBitField()
oldPower := self.Power
newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize))
newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize))
// If below the minimum size requirement, miners have zero power
if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {

84
chain/actors/actor_miner_test.go
View File

@ -3,18 +3,22 @@ package actors_test
import (
"bytes"
"context"
"math"
"math/rand"
"testing"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/rlepluslazy"
hamt "github.com/ipfs/go-hamt-ipld"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/stretchr/testify/assert"
cbg "github.com/whyrusleeping/cbor-gen"
)
@ -50,23 +54,37 @@ func TestMinerCommitSectors(t *testing.T) {
addSectorToMiner(h, t, minerAddr, worker, client, 1)
assertSectorIDs(h, t, minerAddr, []uint64{1})
}
type badRuns struct {
done bool
}
func (br *badRuns) HasNext() bool {
return !br.done
}
func (br *badRuns) NextRun() (rlepluslazy.Run, error) {
br.done = true
return rlepluslazy.Run{true, math.MaxInt64}, nil
}
var _ rlepluslazy.RunIterator = (*badRuns)(nil)
func TestMinerSubmitBadFault(t *testing.T) {
oldSS, oldMin := build.SectorSizes, build.MinimumMinerPower
build.SectorSizes, build.MinimumMinerPower = []uint64{1024}, 1024
defer func() {
build.SectorSizes, build.MinimumMinerPower = oldSS, oldMin
}()
var worker, client address.Address
var minerAddr address.Address
opts := []HarnessOpt{
HarnessAddr(&worker, 1000000),
HarnessAddr(&client, 1000000),
HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid,
func() cbg.CBORMarshaler {
return &actors.StorageMinerConstructorParams{
Owner: worker,
Worker: worker,
SectorSize: 1024,
PeerID: "fakepeerid",
}
}),
HarnessAddMiner(&minerAddr, &worker),
}
h := NewHarness(t, opts...)
@ -92,18 +110,52 @@ func TestMinerSubmitBadFault(t *testing.T) {
ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
ApplyOK(t, ret)
assertSectorIDs(h, t, minerAddr, []uint64{1})
st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
assert.NoError(t, err)
expectedPower := st.Power
if types.BigCmp(expectedPower, types.NewInt(1024)) != 0 {
t.Errorf("Expected power of 1024, got %s", expectedPower)
}
badnum := uint64(0)
badnum--
bf = types.NewBitField()
bf.Set(badnum)
bf.Set(badnum - 1)
ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
ApplyOK(t, ret)
ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
ApplyOK(t, ret)
assertSectorIDs(h, t, minerAddr, []uint64{1})
st, err = getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
assert.NoError(t, err)
currentPower := st.Power
if types.BigCmp(expectedPower, currentPower) != 0 {
t.Errorf("power changed and shouldn't have: %s != %s", expectedPower, currentPower)
}
bf.Set(badnum - 2)
ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
if ret.ExitCode != 3 {
t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
}
assertSectorIDs(h, t, minerAddr, []uint64{1})
rle, err := rlepluslazy.EncodeRuns(&badRuns{}, []byte{})
assert.NoError(t, err)
bf, err = types.NewBitFieldFromBytes(rle)
assert.NoError(t, err)
ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
if ret.ExitCode != 3 {
t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
}
assertSectorIDs(h, t, minerAddr, []uint64{1})
bf = types.NewBitField()
bf.Set(1)
@ -175,7 +227,7 @@ func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint
}
}
func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
func getMinerState(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) (*actors.StorageMinerActorState, error) {
mact, err := st.GetActor(maddr)
if err != nil {
return nil, err
@ -187,6 +239,14 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl
if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
return nil, err
}
return &mstate, nil
}
func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
mstate, err := getMinerState(ctx, st, bs, maddr)
if err != nil {
return nil, err
}
return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors)
}

2
chain/actors/actor_paych.go
View File

@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types.
vb, nerr := sv.SigningBytes()
if nerr != nil {
return nil, aerrors.Escalate(nerr, "failed to serialize signedvoucher")
return nil, aerrors.Absorb(nerr, 1, "failed to serialize signedvoucher")
}
if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil {

14
chain/actors/actor_storagemarket.go
View File

@ -14,10 +14,10 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type StorageMarketActor struct{}
@ -122,7 +122,8 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) {
return nd.Cid(), nil
}
func (sdp *StorageDealProposal) Verify() error {
func (sdp *StorageDealProposal) Verify(worker address.Address) error {
if sdp.Client != worker || worker == address.Undef {
unsigned := *sdp
unsigned.ProposerSignature = nil
var buf bytes.Buffer
@ -130,7 +131,12 @@ func (sdp *StorageDealProposal) Verify() error {
return err
}
return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes())
if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil {
return err
}
}
return nil
}
type OnChainDeal struct {
@ -396,7 +402,7 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe
return aerrors.New(2, "Deals must be submitted by the miner worker")
}
if err := deal.Verify(); err != nil {
if err := deal.Verify(providerWorker); err != nil {
return aerrors.Absorb(err, 3, "verifying proposer signature")
}

2
chain/actors/actor_storagepower.go
View File

@ -67,7 +67,7 @@ type CreateStorageMinerParams struct {
func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) {
if !build.SupportedSectorSize(params.SectorSize) {
return nil, aerrors.New(1, "Unsupported sector size")
return nil, aerrors.Newf(1, "Unsupported sector size: %d", params.SectorSize)
}
var self StoragePowerState

3
chain/actors/aerrors/wrap.go
View File

@ -171,7 +171,8 @@ func HandleExternalError(err error, msg string) ActorError {
}
return &actorError{
fatal: true,
fatal: false,
retCode: 219,
msg: msg,
frame: xerrors.Caller(1),

28
chain/actors/harness2_test.go
View File

@ -115,6 +115,32 @@ func HarnessActor(actor *address.Address, creator *address.Address, code cid.Cid
}
func HarnessAddMiner(addr *address.Address, creator *address.Address) HarnessOpt {
return func(t testing.TB, h *Harness) error {
if h.Stage != HarnessPostInit {
return nil
}
if !addr.Empty() {
return xerrors.New("actor address should be empty")
}
ret, _ := h.InvokeWithValue(t, *creator, actors.StoragePowerAddress,
actors.SPAMethods.CreateStorageMiner, types.NewInt(3000), &actors.StorageMinerConstructorParams{
Owner: *creator,
Worker: *creator,
SectorSize: 1024,
PeerID: "fakepeerid",
})
if ret.ExitCode != 0 {
return xerrors.Errorf("creating actor: %w", ret.ActorErr)
}
var err error
*addr, err = address.NewFromBytes(ret.Return)
return err
}
}
func HarnessCtx(ctx context.Context) HarnessOpt {
return func(t testing.TB, h *Harness) error {
h.ctx = ctx
@ -177,7 +203,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
for _, opt := range options {
err := opt(t, h)
if err != nil {
t.Fatalf("Applying options: %v", err)
t.Fatalf("Applying options: %+v", err)
}
}

2
chain/blocksync/blocksync.go
View File

@ -14,7 +14,7 @@ import (
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
)

2
chain/blocksync/cbor_gen.go
View File

@ -5,7 +5,7 @@ import (
"io"
"github.com/filecoin-project/lotus/chain/types"
cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)

883
chain/deals/cbor_gen.go
View File

@ -1,883 +0,0 @@
package deals
import (
"fmt"
"io"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
var _ = xerrors.Errorf
func (t *AskRequest) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Miner (address.Address) (struct)
if err := t.Miner.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *AskRequest) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Miner (address.Address) (struct)
{
if err := t.Miner.UnmarshalCBOR(br); err != nil {
return err
}
}
return nil
}
func (t *AskResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.Ask (types.SignedStorageAsk) (struct)
if err := t.Ask.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *AskResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Ask (types.SignedStorageAsk) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Ask = new(types.SignedStorageAsk)
if err := t.Ask.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *Proposal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.DealProposal (actors.StorageDealProposal) (struct)
if err := t.DealProposal.MarshalCBOR(w); err != nil {
return err
}
// t.Piece (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Piece); err != nil {
return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
}
return nil
}
func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.DealProposal (actors.StorageDealProposal) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.DealProposal = new(actors.StorageDealProposal)
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
return err
}
}
}
// t.Piece (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
}
t.Piece = c
}
return nil
}
func (t *Response) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{132}); err != nil {
return err
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Message (string) (string)
if len(t.Message) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Message was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Message)); err != nil {
return err
}
// t.Proposal (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Proposal); err != nil {
return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
}
// t.StorageDealSubmission (types.SignedMessage) (struct)
if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *Response) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Message (string) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Message = string(sval)
}
// t.Proposal (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
}
t.Proposal = c
}
// t.StorageDealSubmission (types.SignedMessage) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.StorageDealSubmission = new(types.SignedMessage)
if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *SignedResponse) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Response (deals.Response) (struct)
if err := t.Response.MarshalCBOR(w); err != nil {
return err
}
// t.Signature (types.Signature) (struct)
if err := t.Signature.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Response (deals.Response) (struct)
{
if err := t.Response.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Signature (types.Signature) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Signature = new(types.Signature)
if err := t.Signature.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{136}); err != nil {
return err
}
// t.Data (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Data); err != nil {
return xerrors.Errorf("failed to write cid field t.Data: %w", err)
}
// t.PricePerEpoch (types.BigInt) (struct)
if err := t.PricePerEpoch.MarshalCBOR(w); err != nil {
return err
}
// t.ProposalExpiration (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil {
return err
}
// t.Duration (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil {
return err
}
// t.ProviderAddress (address.Address) (struct)
if err := t.ProviderAddress.MarshalCBOR(w); err != nil {
return err
}
// t.Client (address.Address) (struct)
if err := t.Client.MarshalCBOR(w); err != nil {
return err
}
// t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.MinerID (peer.ID) (string)
if len(t.MinerID) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.MinerID was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.MinerID)); err != nil {
return err
}
return nil
}
func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 8 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Data (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Data: %w", err)
}
t.Data = c
}
// t.PricePerEpoch (types.BigInt) (struct)
{
if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.ProposalExpiration (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ProposalExpiration = uint64(extra)
// t.Duration (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Duration = uint64(extra)
// t.ProviderAddress (address.Address) (struct)
{
if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.Client (address.Address) (struct)
{
if err := t.Client.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.MinerID (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.MinerID = peer.ID(sval)
}
return nil
}
func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{135}); err != nil {
return err
}
// t.ProposalCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
}
// t.Proposal (actors.StorageDealProposal) (struct)
if err := t.Proposal.MarshalCBOR(w); err != nil {
return err
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Miner (peer.ID) (string)
if len(t.Miner) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Miner was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Miner)); err != nil {
return err
}
// t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.PublishMessage (types.SignedMessage) (struct)
if err := t.PublishMessage.MarshalCBOR(w); err != nil {
return err
}
return nil
}
func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 7 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.ProposalCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
}
t.ProposalCid = c
}
// t.Proposal (actors.StorageDealProposal) (struct)
{
if err := t.Proposal.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Miner (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Miner = peer.ID(sval)
}
// t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.PublishMessage (types.SignedMessage) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.PublishMessage = new(types.SignedMessage)
if err := t.PublishMessage.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
}
func (t *MinerDeal) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{135}); err != nil {
return err
}
// t.Client (peer.ID) (string)
if len(t.Client) > cbg.MaxLength {
return xerrors.Errorf("Value in field t.Client was too long")
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Client)); err != nil {
return err
}
// t.Proposal (actors.StorageDealProposal) (struct)
if err := t.Proposal.MarshalCBOR(w); err != nil {
return err
}
// t.ProposalCid (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
}
// t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.Ref (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Ref); err != nil {
return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
return nil
}
func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 7 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Client (peer.ID) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Client = peer.ID(sval)
}
// t.Proposal (actors.StorageDealProposal) (struct)
{
if err := t.Proposal.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.ProposalCid (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
}
t.ProposalCid = c
}
// t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.Ref (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
}
t.Ref = c
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
return nil
}
func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.Proposal (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Proposal); err != nil {
return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
}
// t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
return nil
}
func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.Proposal (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
}
t.Proposal = c
}
// t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
return nil
}

311
chain/deals/client.go
View File

@ -1,311 +0,0 @@
package deals
import (
"context"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
"github.com/libp2p/go-libp2p-core/host"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/retrieval/discovery"
)
var log = logging.Logger("deals")
type ClientDeal struct {
ProposalCid cid.Cid
Proposal actors.StorageDealProposal
State api.DealState
Miner peer.ID
MinerWorker address.Address
DealID uint64
PublishMessage *types.SignedMessage
s inet.Stream
}
type Client struct {
sm *stmgr.StateManager
chain *store.ChainStore
h host.Host
w *wallet.Wallet
// dataTransfer
// TODO: once the data transfer module is complete, the
// client will listen to events on the data transfer module
// Because we are using only a fake DAGService
// implementation, there's no validation or events on the client side
dataTransfer dtypes.ClientDataTransfer
dag dtypes.ClientDAG
discovery *discovery.Local
events *events.Events
fm *market.FundMgr
deals *statestore.StateStore
conns map[cid.Cid]inet.Stream
incoming chan *ClientDeal
updated chan clientDealUpdate
stop chan struct{}
stopped chan struct{}
}
type clientDealUpdate struct {
newState api.DealState
id cid.Cid
err error
mut func(*ClientDeal)
}
type clientApi struct {
full.ChainAPI
full.StateAPI
}
func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client {
c := &Client{
sm: sm,
chain: chain,
h: h,
w: w,
dataTransfer: dataTransfer,
dag: dag,
discovery: discovery,
fm: fm,
events: events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}),
deals: deals,
conns: map[cid.Cid]inet.Stream{},
incoming: make(chan *ClientDeal, 16),
updated: make(chan clientDealUpdate, 16),
stop: make(chan struct{}),
stopped: make(chan struct{}),
}
return c
}
func (c *Client) Run(ctx context.Context) {
go func() {
defer close(c.stopped)
for {
select {
case deal := <-c.incoming:
c.onIncoming(deal)
case update := <-c.updated:
c.onUpdated(ctx, update)
case <-c.stop:
return
}
}
}()
}
func (c *Client) onIncoming(deal *ClientDeal) {
log.Info("incoming deal")
if _, ok := c.conns[deal.ProposalCid]; ok {
log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid)
return
}
c.conns[deal.ProposalCid] = deal.s
if err := c.deals.Begin(deal.ProposalCid, deal); err != nil {
// We may have re-sent the proposal
log.Errorf("deal tracking failed: %s", err)
c.failDeal(deal.ProposalCid, err)
return
}
go func() {
c.updated <- clientDealUpdate{
newState: api.DealUnknown,
id: deal.ProposalCid,
err: nil,
}
}()
}
func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState])
var deal ClientDeal
err := c.deals.Get(update.id).Mutate(func(d *ClientDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
}
deal = *d
return nil
})
if update.err != nil {
log.Errorf("deal %s failed: %s", update.id, update.err)
c.failDeal(update.id, update.err)
return
}
if err != nil {
c.failDeal(update.id, err)
return
}
switch update.newState {
case api.DealUnknown: // new
c.handle(ctx, deal, c.new, api.DealAccepted)
case api.DealAccepted:
c.handle(ctx, deal, c.accepted, api.DealStaged)
case api.DealStaged:
c.handle(ctx, deal, c.staged, api.DealSealing)
case api.DealSealing:
c.handle(ctx, deal, c.sealing, api.DealNoUpdate)
// TODO: DealComplete -> watch for faults, expiration, etc.
}
}
type ClientDealProposal struct {
Data cid.Cid
PricePerEpoch types.BigInt
ProposalExpiration uint64
Duration uint64
ProviderAddress address.Address
Client address.Address
MinerWorker address.Address
MinerID peer.ID
}
func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) {
if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil {
return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err)
}
commP, pieceSize, err := c.commP(ctx, p.Data)
if err != nil {
return cid.Undef, xerrors.Errorf("computing commP failed: %w", err)
}
dealProposal := &actors.StorageDealProposal{
PieceRef: commP,
PieceSize: uint64(pieceSize),
Client: p.Client,
Provider: p.ProviderAddress,
ProposalExpiration: p.ProposalExpiration,
Duration: p.Duration,
StoragePricePerEpoch: p.PricePerEpoch,
StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc
}
if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil {
return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err)
}
proposalNd, err := cborutil.AsIpld(dealProposal)
if err != nil {
return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err)
}
s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
if err != nil {
return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
}
proposal := &Proposal{
DealProposal: dealProposal,
Piece: p.Data,
}
if err := cborutil.WriteCborRPC(s, proposal); err != nil {
s.Reset()
return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err)
}
deal := &ClientDeal{
ProposalCid: proposalNd.Cid(),
Proposal: *dealProposal,
State: api.DealUnknown,
Miner: p.MinerID,
MinerWorker: p.MinerWorker,
s: s,
}
c.incoming <- deal
return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{
Address: dealProposal.Provider,
ID: deal.Miner,
})
}
func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) {
s, err := c.h.NewStream(ctx, p, AskProtocolID)
if err != nil {
return nil, xerrors.Errorf("failed to open stream to miner: %w", err)
}
req := &AskRequest{
Miner: a,
}
if err := cborutil.WriteCborRPC(s, req); err != nil {
return nil, xerrors.Errorf("failed to send ask request: %w", err)
}
var out AskResponse
if err := cborutil.ReadCborRPC(s, &out); err != nil {
return nil, xerrors.Errorf("failed to read ask response: %w", err)
}
if out.Ask == nil {
return nil, xerrors.Errorf("got no ask back")
}
if out.Ask.Ask.Miner != a {
return nil, xerrors.Errorf("got back ask for wrong miner")
}
if err := c.checkAskSignature(out.Ask); err != nil {
return nil, xerrors.Errorf("ask was not properly signed")
}
return out.Ask, nil
}
func (c *Client) List() ([]ClientDeal, error) {
var out []ClientDeal
if err := c.deals.List(&out); err != nil {
return nil, err
}
return out, nil
}
func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
var out ClientDeal
if err := c.deals.Get(d).Get(&out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) Stop() {
close(c.stop)
<-c.stopped
}

233
chain/deals/client_states.go
View File

@ -1,233 +0,0 @@
package deals
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
)
type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error)
func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) {
go func() {
mut, err := cb(ctx, deal)
if err != nil {
next = api.DealError
}
if err == nil && next == api.DealNoUpdate {
return
}
select {
case c.updated <- clientDealUpdate{
newState: next,
id: deal.ProposalCid,
err: err,
mut: mut,
}:
case <-c.stop:
}
}()
}
func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
resp, err := c.readStorageDealResp(deal)
if err != nil {
return nil, err
}
// TODO: verify StorageDealSubmission
if err := c.disconnect(deal); err != nil {
return nil, err
}
/* data transfer happens */
if resp.State != api.DealAccepted {
return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
}
return func(info *ClientDeal) {
info.PublishMessage = resp.StorageDealSubmission
}, nil
}
func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
log.Infow("DEAL ACCEPTED!")
pubmsg := deal.PublishMessage.Message
pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
if err != nil {
return nil, xerrors.Errorf("getting miner worker failed: %w", err)
}
if pubmsg.From != pw {
return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
}
if pubmsg.To != actors.StorageMarketAddress {
return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
}
if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
}
var params actors.PublishStorageDealsParams
if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
return nil, err
}
dealIdx := -1
for i, storageDeal := range params.Deals {
// TODO: make it less hacky
sd := storageDeal
eq, err := cborutil.Equals(&deal.Proposal, &sd)
if err != nil {
return nil, err
}
if eq {
dealIdx = i
break
}
}
if dealIdx == -1 {
return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid())
}
// TODO: timeout
_, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid())
if err != nil {
return nil, xerrors.Errorf("waiting for deal publish message: %w", err)
}
if ret.ExitCode != 0 {
return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
}
var res actors.PublishStorageDealResponse
if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
return nil, err
}
return func(info *ClientDeal) {
info.DealID = res.DealIDs[dealIdx]
}, nil
}
func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
// TODO: Maybe wait for pre-commit
return nil, nil
}
func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
if err != nil {
// TODO: This may be fine for some errors
return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
if sd.ActivationEpoch > 0 {
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
}:
case <-c.stop:
}
return true, false, nil
}
return false, true, nil
}
called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) {
defer func() {
if err != nil {
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
err: xerrors.Errorf("handling applied event: %w", err),
}:
case <-c.stop:
}
}
}()
if msg == nil {
log.Error("timed out waiting for deal activation... what now?")
return false, nil
}
sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
if err != nil {
return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
if sd.ActivationEpoch == 0 {
return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height())
}
log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch)
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
}:
case <-c.stop:
}
return false, nil
}
revert := func(ctx context.Context, ts *types.TipSet) error {
log.Warn("deal activation reverted; TODO: actually handle this!")
// TODO: Just go back to DealSealing?
return nil
}
matchEvent := func(msg *types.Message) (bool, error) {
if msg.To != deal.Proposal.Provider {
return false, nil
}
if msg.Method != actors.MAMethods.ProveCommitSector {
return false, nil
}
var params actors.SectorProveCommitInfo
if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
return false, err
}
var found bool
for _, dealID := range params.DealIDs {
if dealID == deal.DealID {
found = true
break
}
}
return found, nil
}
if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
return nil, xerrors.Errorf("failed to set up called handler")
}
return nil, nil
}

170
chain/deals/client_utils.go
View File

@ -1,170 +0,0 @@
package deals
import (
"bytes"
"context"
"runtime"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
unixfile "github.com/ipfs/go-unixfs/file"
"github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func (c *Client) failDeal(id cid.Cid, cerr error) {
if cerr == nil {
_, f, l, _ := runtime.Caller(1)
cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
}
s, ok := c.conns[id]
if ok {
_ = s.Reset()
delete(c.conns, id)
}
// TODO: store in some sort of audit log
log.Errorf("deal %s failed: %+v", id, cerr)
}
func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) {
root, err := c.dag.Get(ctx, data)
if err != nil {
log.Errorf("failed to get file root for deal: %s", err)
return nil, 0, err
}
n, err := unixfile.NewUnixfsFile(ctx, c.dag, root)
if err != nil {
log.Errorf("cannot open unixfs file: %s", err)
return nil, 0, err
}
uf, ok := n.(files.File)
if !ok {
// TODO: we probably got directory, how should we handle this in unixfs mode?
return nil, 0, xerrors.New("unsupported unixfs type")
}
s, err := uf.Size()
if err != nil {
return nil, 0, err
}
pr, psize := padreader.New(uf, uint64(s))
commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize)
if err != nil {
return nil, 0, xerrors.Errorf("generating CommP: %w", err)
}
return commp[:], psize, nil
}
func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
s, ok := c.conns[deal.ProposalCid]
if !ok {
// TODO: Try to re-establish the connection using query protocol
return nil, xerrors.Errorf("no connection to miner")
}
var resp SignedResponse
if err := cborutil.ReadCborRPC(s, &resp); err != nil {
log.Errorw("failed to read Response message", "error", err)
return nil, err
}
if err := resp.Verify(deal.MinerWorker); err != nil {
return nil, xerrors.Errorf("verifying response signature failed", err)
}
if resp.Response.Proposal != deal.ProposalCid {
return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid)
}
return &resp.Response, nil
}
func (c *Client) disconnect(deal ClientDeal) error {
s, ok := c.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(c.conns, deal.ProposalCid)
return err
}
var _ datatransfer.RequestValidator = &ClientRequestValidator{}
// ClientRequestValidator validates data transfer requests for the client
// in a storage market
type ClientRequestValidator struct {
deals *statestore.StateStore
}
// NewClientRequestValidator returns a new client request validator for the
// given datastore
func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator {
crv := &ClientRequestValidator{
deals: deals,
}
return crv
}
// ValidatePush validates a push request received from the peer that will send data
// Will always error because clients should not accept push requests from a provider
// in a storage deal (i.e. send data to client).
func (c *ClientRequestValidator) ValidatePush(
sender peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
return ErrNoPushAccepted
}
// ValidatePull validates a pull request received from the peer that will receive data
// Will succeed only if:
// - voucher has correct type
// - voucher references an active deal
// - referenced deal matches the receiver (miner)
// - referenced deal matches the given base CID
// - referenced deal is in an acceptable state
func (c *ClientRequestValidator) ValidatePull(
receiver peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
if !ok {
return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
}
var deal ClientDeal
err := c.deals.Get(dealVoucher.Proposal).Get(&deal)
if err != nil {
return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
}
if deal.Miner != receiver {
return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer)
}
if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
}
for _, state := range DataTransferStates {
if deal.State == state {
return nil
}
}
return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
}

294
chain/deals/provider.go
View File

@ -1,294 +0,0 @@
package deals
import (
"context"
"errors"
"sync"
cid "github.com/ipfs/go-cid"
datastore "github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
var ProviderDsPrefix = "/deals/provider"
type MinerDeal struct {
Client peer.ID
Proposal actors.StorageDealProposal
ProposalCid cid.Cid
State api.DealState
Ref cid.Cid
DealID uint64
SectorID uint64 // Set when State >= DealStaged
s inet.Stream
}
type Provider struct {
pricePerByteBlock types.BigInt // how much we want for storing one byte for one block
minPieceSize uint64
ask *types.SignedStorageAsk
askLk sync.Mutex
secb *sectorblocks.SectorBlocks
sminer *storage.Miner
full api.FullNode
// TODO: This will go away once storage market module + CAR
// is implemented
dag dtypes.StagingDAG
// dataTransfer is the manager of data transfers used by this storage provider
dataTransfer dtypes.ProviderDataTransfer
deals *statestore.StateStore
ds dtypes.MetadataDS
conns map[cid.Cid]inet.Stream
actor address.Address
incoming chan MinerDeal
updated chan minerDealUpdate
stop chan struct{}
stopped chan struct{}
}
type minerDealUpdate struct {
newState api.DealState
id cid.Cid
err error
mut func(*MinerDeal)
}
var (
// ErrDataTransferFailed means a data transfer for a deal failed
ErrDataTransferFailed = errors.New("deal data transfer failed")
)
func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, fullNode api.FullNode) (*Provider, error) {
addr, err := ds.Get(datastore.NewKey("miner-address"))
if err != nil {
return nil, err
}
minerAddress, err := address.NewFromBytes(addr)
if err != nil {
return nil, err
}
h := &Provider{
sminer: sminer,
dag: dag,
dataTransfer: dataTransfer,
full: fullNode,
secb: secb,
pricePerByteBlock: types.NewInt(3), // TODO: allow setting
minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up))
conns: map[cid.Cid]inet.Stream{},
incoming: make(chan MinerDeal),
updated: make(chan minerDealUpdate),
stop: make(chan struct{}),
stopped: make(chan struct{}),
actor: minerAddress,
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))),
ds: ds,
}
if err := h.tryLoadAsk(); err != nil {
return nil, err
}
if h.ask == nil {
// TODO: we should be fine with this state, and just say it means 'not actively accepting deals'
// for now... lets just set a price
if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil {
return nil, xerrors.Errorf("failed setting a default price: %w", err)
}
}
// register a data transfer event handler -- this will move deals from
// accepted to staged
h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent)
return h, nil
}
func (p *Provider) Run(ctx context.Context) {
// TODO: restore state
go func() {
defer log.Warn("quitting deal provider loop")
defer close(p.stopped)
for {
select {
case deal := <-p.incoming: // DealAccepted
p.onIncoming(deal)
case update := <-p.updated: // DealStaged
p.onUpdated(ctx, update)
case <-p.stop:
return
}
}
}()
}
func (p *Provider) onIncoming(deal MinerDeal) {
log.Info("incoming deal")
p.conns[deal.ProposalCid] = deal.s
if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil {
// This can happen when client re-sends proposal
p.failDeal(deal.ProposalCid, err)
log.Errorf("deal tracking failed: %s", err)
return
}
go func() {
p.updated <- minerDealUpdate{
newState: api.DealAccepted,
id: deal.ProposalCid,
err: nil,
}
}()
}
func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState])
if update.err != nil {
log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err)
p.failDeal(update.id, update.err)
return
}
var deal MinerDeal
err := p.deals.Get(update.id).Mutate(func(d *MinerDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
}
deal = *d
return nil
})
if err != nil {
p.failDeal(update.id, err)
return
}
switch update.newState {
case api.DealAccepted:
p.handle(ctx, deal, p.accept, api.DealNoUpdate)
case api.DealStaged:
p.handle(ctx, deal, p.staged, api.DealSealing)
case api.DealSealing:
p.handle(ctx, deal, p.sealing, api.DealComplete)
case api.DealComplete:
p.handle(ctx, deal, p.complete, api.DealNoUpdate)
}
}
// onDataTransferEvent is the function called when an event occurs in a data
// transfer -- it reads the voucher to verify this even occurred in a storage
// market deal, then, based on the data transfer event that occurred, it generates
// and update message for the deal -- either moving to staged for a completion
// event or moving to error if a data transfer error occurs
func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) {
voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher)
// if this event is for a transfer not related to storage, ignore
if !ok {
return
}
// data transfer events for opening and progress do not affect deal state
var next api.DealState
var err error
var mut func(*MinerDeal)
switch event {
case datatransfer.Complete:
next = api.DealStaged
mut = func(deal *MinerDeal) {
deal.DealID = voucher.DealID
}
case datatransfer.Error:
next = api.DealFailed
err = ErrDataTransferFailed
default:
// the only events we care about are complete and error
return
}
select {
case p.updated <- minerDealUpdate{
newState: next,
id: voucher.Proposal,
err: err,
mut: mut,
}:
case <-p.stop:
}
}
func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
proposalNd, err := cborutil.AsIpld(proposal.DealProposal)
if err != nil {
return MinerDeal{}, err
}
return MinerDeal{
Client: s.Conn().RemotePeer(),
Proposal: *proposal.DealProposal,
ProposalCid: proposalNd.Cid(),
State: api.DealUnknown,
Ref: proposal.Piece,
s: s,
}, nil
}
func (p *Provider) HandleStream(s inet.Stream) {
log.Info("Handling storage deal proposal!")
proposal, err := p.readProposal(s)
if err != nil {
log.Error(err)
s.Close()
return
}
deal, err := p.newDeal(s, proposal)
if err != nil {
log.Errorf("%+v", err)
s.Close()
return
}
p.incoming <- deal
}
func (p *Provider) Stop() {
close(p.stop)
<-p.stopped
}

160
chain/deals/provider_asks.go
View File

@ -1,160 +0,0 @@
package deals
import (
"bytes"
"context"
"time"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
datastore "github.com/ipfs/go-datastore"
inet "github.com/libp2p/go-libp2p-core/network"
"golang.org/x/xerrors"
)
func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error {
p.askLk.Lock()
defer p.askLk.Unlock()
var seqno uint64
if p.ask != nil {
seqno = p.ask.Ask.SeqNo + 1
}
now := time.Now().Unix()
ask := &types.StorageAsk{
Price: price,
Timestamp: uint64(now),
Expiry: uint64(now + ttlsecs),
Miner: p.actor,
SeqNo: seqno,
MinPieceSize: p.minPieceSize,
}
ssa, err := p.signAsk(ask)
if err != nil {
return err
}
return p.saveAsk(ssa)
}
func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk {
p.askLk.Lock()
defer p.askLk.Unlock()
if m != p.actor {
return nil
}
return p.ask
}
func (p *Provider) HandleAskStream(s inet.Stream) {
defer s.Close()
var ar AskRequest
if err := cborutil.ReadCborRPC(s, &ar); err != nil {
log.Errorf("failed to read AskRequest from incoming stream: %s", err)
return
}
resp := p.processAskRequest(&ar)
if err := cborutil.WriteCborRPC(s, resp); err != nil {
log.Errorf("failed to write ask response: %s", err)
return
}
}
func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse {
return &AskResponse{
Ask: p.getAsk(ar.Miner),
}
}
var bestAskKey = datastore.NewKey("latest-ask")
func (p *Provider) tryLoadAsk() error {
p.askLk.Lock()
defer p.askLk.Unlock()
err := p.loadAsk()
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
log.Warn("no previous ask found, miner will not accept deals until a price is set")
return nil
}
return err
}
return nil
}
func (p *Provider) loadAsk() error {
askb, err := p.ds.Get(datastore.NewKey("latest-ask"))
if err != nil {
return xerrors.Errorf("failed to load most recent ask from disk: %w", err)
}
var ssa types.SignedStorageAsk
if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
return err
}
p.ask = &ssa
return nil
}
func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) {
b, err := cborutil.Dump(a)
if err != nil {
return nil, err
}
worker, err := p.getWorker(p.actor)
if err != nil {
return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err)
}
sig, err := p.full.WalletSign(context.TODO(), worker, b)
if err != nil {
return nil, err
}
return &types.SignedStorageAsk{
Ask: a,
Signature: sig,
}, nil
}
func (p *Provider) saveAsk(a *types.SignedStorageAsk) error {
b, err := cborutil.Dump(a)
if err != nil {
return err
}
if err := p.ds.Put(bestAskKey, b); err != nil {
return err
}
p.ask = a
return nil
}
func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error {
tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState()
w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner)
if err != nil {
return xerrors.Errorf("failed to get worker for miner in ask", err)
}
sigb, err := cborutil.Dump(ask.Ask)
if err != nil {
return xerrors.Errorf("failed to re-serialize ask")
}
return ask.Signature.Verify(w, sigb)
}

215
chain/deals/provider_states.go
View File

@ -1,215 +0,0 @@
package deals
import (
"bytes"
"context"
ipldfree "github.com/ipld/go-ipld-prime/impl/free"
"github.com/ipld/go-ipld-prime/traversal/selector"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
unixfile "github.com/ipfs/go-unixfs/file"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error)
func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) {
go func() {
mut, err := cb(ctx, deal)
if err == nil && next == api.DealNoUpdate {
return
}
select {
case p.updated <- minerDealUpdate{
newState: next,
id: deal.ProposalCid,
err: err,
mut: mut,
}:
case <-p.stop:
}
}()
}
// ACCEPTED
func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
head, err := p.full.ChainHead(ctx)
if err != nil {
return nil, err
}
if head.Height() >= deal.Proposal.ProposalExpiration {
return nil, xerrors.Errorf("deal proposal already expired")
}
// TODO: check StorageCollateral
minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30))
if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) {
return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice)
}
if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize {
return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize)
}
// check market funds
clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil)
if err != nil {
return nil, xerrors.Errorf("getting client market balance failed: %w", err)
}
// This doesn't guarantee that the client won't withdraw / lock those funds
// but it's a decent first filter
if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) {
return nil, xerrors.New("clientMarketBalance.Available too small")
}
waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil)
if err != nil {
return nil, err
}
// TODO: check StorageCollateral (may be too large (or too small))
if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil {
return nil, err
}
log.Info("publishing deal")
params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
Deals: []actors.StorageDealProposal{deal.Proposal},
})
if err != nil {
return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
}
// TODO: We may want this to happen after fetching data
smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: waddr,
Value: types.NewInt(0),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.PublishStorageDeals,
Params: params,
})
if err != nil {
return nil, err
}
r, err := p.full.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return nil, err
}
if r.Receipt.ExitCode != 0 {
return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)
}
var resp actors.PublishStorageDealResponse
if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
return nil, err
}
if len(resp.DealIDs) != 1 {
return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA")
}
log.Infof("fetching data for a deal %d", resp.DealIDs[0])
err = p.sendSignedResponse(&Response{
State: api.DealAccepted,
Proposal: deal.ProposalCid,
StorageDealSubmission: smsg,
})
if err != nil {
return nil, err
}
if err := p.disconnect(deal); err != nil {
log.Warnf("closing client connection: %+v", err)
}
ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder())
// this is the selector for "get the whole DAG"
// TODO: support storage deals with custom payload selectors
allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
// initiate a pull data transfer. This will complete asynchronously and the
// completion of the data transfer will trigger a change in deal state
// (see onDataTransferEvent)
_, err = p.dataTransfer.OpenPullDataChannel(ctx,
deal.Client,
&StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]},
deal.Ref,
allSelector,
)
if err != nil {
return nil, xerrors.Errorf("failed to open pull data channel: %w", err)
}
return nil, nil
}
// STAGED
func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
root, err := p.dag.Get(ctx, deal.Ref)
if err != nil {
return nil, xerrors.Errorf("failed to get file root for deal: %s", err)
}
// TODO: abstract this away into ReadSizeCloser + implement different modes
n, err := unixfile.NewUnixfsFile(ctx, p.dag, root)
if err != nil {
return nil, xerrors.Errorf("cannot open unixfs file: %s", err)
}
uf, ok := n.(sectorblocks.UnixfsReader)
if !ok {
// we probably got directory, unsupported for now
return nil, xerrors.Errorf("unsupported unixfs file type")
}
// TODO: uf.Size() is user input, not trusted
// This won't be useful / here after we migrate to putting CARs into sectors
size, err := uf.Size()
if err != nil {
return nil, xerrors.Errorf("getting unixfs file size: %w", err)
}
if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
}
sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
if err != nil {
return nil, xerrors.Errorf("AddPiece failed: %s", err)
}
log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
return func(deal *MinerDeal) {
deal.SectorID = sectorID
}, nil
}
// SEALING
func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
// TODO: consider waiting for seal to happen
return nil, nil
}
func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
// TODO: observe sector lifecycle, status, expiration..
return nil, nil
}

198
chain/deals/provider_utils.go
View File

@ -1,198 +0,0 @@
package deals
import (
"bytes"
"context"
"runtime"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/ipld/go-ipld-prime"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/ipfs/go-cid"
inet "github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
)
func (p *Provider) failDeal(id cid.Cid, cerr error) {
if err := p.deals.Get(id).End(); err != nil {
log.Warnf("deals.End: %s", err)
}
if cerr == nil {
_, f, l, _ := runtime.Caller(1)
cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
}
log.Warnf("deal %s failed: %s", id, cerr)
err := p.sendSignedResponse(&Response{
State: api.DealFailed,
Message: cerr.Error(),
Proposal: id,
})
s, ok := p.conns[id]
if ok {
_ = s.Reset()
delete(p.conns, id)
}
if err != nil {
log.Warnf("notifying client about deal failure: %s", err)
}
}
func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) {
if err := cborutil.ReadCborRPC(s, &proposal); err != nil {
log.Errorw("failed to read proposal message", "error", err)
return proposal, err
}
if err := proposal.DealProposal.Verify(); err != nil {
return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err)
}
if proposal.DealProposal.Provider != p.actor {
log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider)
return proposal, err
}
return
}
func (p *Provider) sendSignedResponse(resp *Response) error {
s, ok := p.conns[resp.Proposal]
if !ok {
return xerrors.New("couldn't send response: not connected")
}
msg, err := cborutil.Dump(resp)
if err != nil {
return xerrors.Errorf("serializing response: %w", err)
}
worker, err := p.getWorker(p.actor)
if err != nil {
return err
}
sig, err := p.full.WalletSign(context.TODO(), worker, msg)
if err != nil {
return xerrors.Errorf("failed to sign response message: %w", err)
}
signedResponse := &SignedResponse{
Response: *resp,
Signature: sig,
}
err = cborutil.WriteCborRPC(s, signedResponse)
if err != nil {
// Assume client disconnected
s.Close()
delete(p.conns, resp.Proposal)
}
return err
}
func (p *Provider) disconnect(deal MinerDeal) error {
s, ok := p.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(p.conns, deal.ProposalCid)
return err
}
func (p *Provider) getWorker(miner address.Address) (address.Address, error) {
getworker := &types.Message{
To: miner,
From: miner,
Method: actors.MAMethods.GetWorkerAddr,
}
r, err := p.full.StateCall(context.TODO(), getworker, nil)
if err != nil {
return address.Undef, xerrors.Errorf("getting worker address: %w", err)
}
if r.ExitCode != 0 {
return address.Undef, xerrors.Errorf("getWorker call failed: %d", r.ExitCode)
}
return address.NewFromBytes(r.Return)
}
var _ datatransfer.RequestValidator = &ProviderRequestValidator{}
// ProviderRequestValidator validates data transfer requests for the provider
// in a storage market
type ProviderRequestValidator struct {
deals *statestore.StateStore
}
// NewProviderRequestValidator returns a new client request validator for the
// given datastore
func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator {
return &ProviderRequestValidator{
deals: deals,
}
}
// ValidatePush validates a push request received from the peer that will send data
// Will succeed only if:
// - voucher has correct type
// - voucher references an active deal
// - referenced deal matches the client
// - referenced deal matches the given base CID
// - referenced deal is in an acceptable state
func (m *ProviderRequestValidator) ValidatePush(
sender peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
if !ok {
return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
}
var deal MinerDeal
err := m.deals.Get(dealVoucher.Proposal).Get(&deal)
if err != nil {
return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
}
if deal.Client != sender {
return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer)
}
if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
}
for _, state := range DataTransferStates {
if deal.State == state {
return nil
}
}
return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
}
// ValidatePull validates a pull request received from the peer that will receive data.
// Will always error because providers should not accept pull requests from a client
// in a storage deal (i.e. send data to client).
func (m *ProviderRequestValidator) ValidatePull(
receiver peer.ID,
voucher datatransfer.Voucher,
baseCid cid.Cid,
Selector ipld.Node) error {
return ErrNoPullAccepted
}

291
chain/deals/request_validation_test.go
View File

@ -1,291 +0,0 @@
package deals_test
import (
"fmt"
"math/rand"
"testing"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
dss "github.com/ipfs/go-datastore/sync"
blocksutil "github.com/ipfs/go-ipfs-blocksutil"
"github.com/libp2p/go-libp2p-core/peer"
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statestore"
)
var blockGenerator = blocksutil.NewBlockGenerator()
type wrongDTType struct {
}
func (wrongDTType) ToBytes() ([]byte, error) {
return []byte{}, nil
}
func (wrongDTType) FromBytes([]byte) error {
return fmt.Errorf("not implemented")
}
func (wrongDTType) Identifier() string {
return "WrongDTTYPE"
}
func uniqueStorageDealProposal() (actors.StorageDealProposal, error) {
clientAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return actors.StorageDealProposal{}, err
}
providerAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return actors.StorageDealProposal{}, err
}
return actors.StorageDealProposal{
PieceRef: blockGenerator.Next().Cid().Bytes(),
Client: clientAddr,
Provider: providerAddr,
ProposerSignature: &types.Signature{
Data: []byte("foo bar cat dog"),
Type: types.KTBLS,
},
}, nil
}
func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) {
newProposal, err := uniqueStorageDealProposal()
if err != nil {
return deals.ClientDeal{}, err
}
proposalNd, err := cborutil.AsIpld(&newProposal)
if err != nil {
return deals.ClientDeal{}, err
}
minerAddr, err := address.NewIDAddress(uint64(rand.Int()))
if err != nil {
return deals.ClientDeal{}, err
}
return deals.ClientDeal{
Proposal: newProposal,
ProposalCid: proposalNd.Cid(),
Miner: minerID,
MinerWorker: minerAddr,
State: state,
}, nil
}
func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) {
newProposal, err := uniqueStorageDealProposal()
if err != nil {
return deals.MinerDeal{}, err
}
proposalNd, err := cborutil.AsIpld(&newProposal)
if err != nil {
return deals.MinerDeal{}, err
}
ref, err := cid.Cast(newProposal.PieceRef)
if err != nil {
return deals.MinerDeal{}, err
}
return deals.MinerDeal{
Proposal: newProposal,
ProposalCid: proposalNd.Cid(),
Client: clientID,
State: state,
Ref: ref,
}, nil
}
func TestClientRequestValidation(t *testing.T) {
ds := dss.MutexWrap(datastore.NewMapDatastore())
state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
crv := deals.NewClientRequestValidator(state)
minerID := peer.ID("fakepeerid")
block := blockGenerator.Next()
t.Run("ValidatePush fails", func(t *testing.T) {
if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) {
t.Fatal("Push should fail for the client request validator for storage deals")
}
})
t.Run("ValidatePull fails deal not found", func(t *testing.T) {
proposal, err := uniqueStorageDealProposal()
if err != nil {
t.Fatal("error creating proposal")
}
proposalNd, err := cborutil.AsIpld(&proposal)
if err != nil {
t.Fatal("error serializing proposal")
}
pieceRef, err := cid.Cast(proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Pull should fail if there is no deal stored")
}
})
t.Run("ValidatePull fails wrong client", func(t *testing.T) {
otherMiner := peer.ID("otherminer")
clientDeal, err := newClientDeal(otherMiner, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
t.Fatal("Pull should fail if miner address is incorrect")
}
})
t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Pull should fail if piece ref is incorrect")
}
})
t.Run("ValidatePull fails wrong deal state", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealComplete)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
t.Fatal("Pull should fail if deal is in a state that cannot be data transferred")
}
})
t.Run("ValidatePull succeeds", func(t *testing.T) {
clientDeal, err := newClientDeal(minerID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil {
t.Fatal("Pull should should succeed when all parameters are correct")
}
})
}
func TestProviderRequestValidation(t *testing.T) {
ds := dss.MutexWrap(datastore.NewMapDatastore())
state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
mrv := deals.NewProviderRequestValidator(state)
clientID := peer.ID("fakepeerid")
block := blockGenerator.Next()
t.Run("ValidatePull fails", func(t *testing.T) {
if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) {
t.Fatal("Pull should fail for the provider request validator for storage deals")
}
})
t.Run("ValidatePush fails deal not found", func(t *testing.T) {
proposal, err := uniqueStorageDealProposal()
if err != nil {
t.Fatal("error creating proposal")
}
proposalNd, err := cborutil.AsIpld(&proposal)
if err != nil {
t.Fatal("error serializing proposal")
}
pieceRef, err := cid.Cast(proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Push should fail if there is no deal stored")
}
})
t.Run("ValidatePush fails wrong miner", func(t *testing.T) {
otherClient := peer.ID("otherclient")
minerDeal, err := newMinerDeal(otherClient, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
t.Fatal("Push should fail if miner address is incorrect")
}
})
t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Push should fail if piece ref is incorrect")
}
})
t.Run("ValidatePush fails wrong deal state", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealComplete)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
t.Fatal("Push should fail if deal is in a state that cannot be data transferred")
}
})
t.Run("ValidatePush succeeds", func(t *testing.T) {
minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
if err != nil {
t.Fatal("error creating client deal")
}
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
if err != nil {
t.Fatal("unable to construct piece cid")
}
if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil {
t.Fatal("Push should should succeed when all parameters are correct")
}
})
}

114
chain/deals/types.go
View File

@ -1,114 +0,0 @@
package deals
import (
"bytes"
"errors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
)
var (
// ErrWrongVoucherType means the voucher was not the correct type can validate against
ErrWrongVoucherType = errors.New("cannot validate voucher type.")
// ErrNoPushAccepted just means clients do not accept pushes for storage deals
ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.")
// ErrNoPullAccepted just means providers do not accept pulls for storage deals
ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.")
// ErrNoDeal means no active deal was found for this vouchers proposal cid
ErrNoDeal = errors.New("no deal found for this proposal.")
// ErrWrongPeer means that the other peer for this data transfer request does not match
// the other peer for the deal
ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.")
// ErrWrongPiece means that the pieceref for this data transfer request does not match
// the one specified in the deal
ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.")
// ErrInacceptableDealState means the deal for this transfer is not in a deal state
// where transfer can be performed
ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.")
// DataTransferStates are the states in which it would make sense to actually start a data transfer
DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown}
)
const DealProtocolID = "/fil/storage/mk/1.0.1"
const AskProtocolID = "/fil/storage/ask/1.0.1"
type Proposal struct {
DealProposal *actors.StorageDealProposal
Piece cid.Cid // Used for retrieving from the client
}
type Response struct {
State api.DealState
// DealProposalRejected
Message string
Proposal cid.Cid
// DealAccepted
StorageDealSubmission *types.SignedMessage
}
// TODO: Do we actually need this to be signed?
type SignedResponse struct {
Response Response
Signature *types.Signature
}
func (r *SignedResponse) Verify(addr address.Address) error {
b, err := cborutil.Dump(&r.Response)
if err != nil {
return err
}
return r.Signature.Verify(addr, b)
}
type AskRequest struct {
Miner address.Address
}
type AskResponse struct {
Ask *types.SignedStorageAsk
}
// StorageDataTransferVoucher is the voucher type for data transfers
// used by the storage market
type StorageDataTransferVoucher struct {
Proposal cid.Cid
DealID uint64
}
// ToBytes converts the StorageDataTransferVoucher to raw bytes
func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) {
var buf bytes.Buffer
err := dv.MarshalCBOR(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// FromBytes converts the StorageDataTransferVoucher to raw bytes
func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error {
r := bytes.NewReader(raw)
return dv.UnmarshalCBOR(r)
}
// Identifier is the unique string identifier for a StorageDataTransferVoucher
func (dv *StorageDataTransferVoucher) Identifier() string {
return "StorageDataTransferVoucher"
}

2
chain/events/events.go
View File

@ -6,7 +6,7 @@ import (
"time"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"

4
chain/gen/gen.go
View File

@ -11,6 +11,7 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-car"
offline "github.com/ipfs/go-ipfs-exchange-offline"
@ -28,13 +29,12 @@ import (
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/repo"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
)
var log = logging.Logger("gen")

12
chain/messagepool/messagepool.go
View File

@ -13,7 +13,7 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"github.com/ipfs/go-datastore/query"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
pubsub "github.com/libp2p/go-libp2p-pubsub"
lps "github.com/whyrusleeping/pubsub"
"go.uber.org/multierr"
@ -263,24 +263,24 @@ func (mp *MessagePool) addLocal(m *types.SignedMessage, msgb []byte) error {
return nil
}
func (mp *MessagePool) Push(m *types.SignedMessage) error {
func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) {
msgb, err := m.Serialize()
if err != nil {
return err
return cid.Undef, err
}
if err := mp.Add(m); err != nil {
return err
return cid.Undef, err
}
mp.lk.Lock()
if err := mp.addLocal(m, msgb); err != nil {
mp.lk.Unlock()
return err
return cid.Undef, err
}
mp.lk.Unlock()
return mp.api.PubSubPublish(msgTopic, msgb)
return m.Cid(), mp.api.PubSubPublish(msgTopic, msgb)
}
func (mp *MessagePool) Add(m *types.SignedMessage) error {

2
chain/metrics/consensus.go
View File

@ -6,7 +6,7 @@ import (
"time"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"go.uber.org/fx"

2
chain/state/statetree.go
View File

@ -6,7 +6,7 @@ import (
"github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"

2
chain/stmgr/stmgr.go
View File

@ -18,7 +18,7 @@ import (
bls "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
)

3
chain/stmgr/utils.go
View File

@ -4,12 +4,13 @@ import (
"context"
ffi "github.com/filecoin-project/filecoin-ffi"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
amt "github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"

2
chain/store/store.go
View File

@ -23,7 +23,7 @@ import (
dstore "github.com/ipfs/go-datastore"
hamt "github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"
pubsub "github.com/whyrusleeping/pubsub"
"golang.org/x/xerrors"

8
chain/sub/incoming.go
View File

@ -4,10 +4,11 @@ import (
"context"
"time"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
connmgr "github.com/libp2p/go-libp2p-core/connmgr"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/messagepool"
"github.com/filecoin-project/lotus/chain/types"
@ -33,6 +34,11 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
continue
}
if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit {
log.Warnf("received block with too many messages over pubsub")
continue
}
go func() {
log.Infof("New block over pubsub: %s", blk.Cid())

23
chain/sync.go
View File

@ -12,12 +12,13 @@ import (
"github.com/Gurpartap/async"
bls "github.com/filecoin-project/filecoin-ffi"
amt "github.com/filecoin-project/go-amt-ipld"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
hamt "github.com/ipfs/go-hamt-ipld"
bstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/connmgr"
"github.com/libp2p/go-libp2p-core/peer"
cbg "github.com/whyrusleeping/cbor-gen"
@ -26,6 +27,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
@ -35,7 +37,6 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("chain")
@ -188,6 +189,10 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe
}
func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit {
return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc)
}
var bcids, scids []cbg.CBORMarshaler
for _, m := range fblk.BlsMessages {
c := cbg.CborCid(m.Cid())
@ -294,6 +299,10 @@ func zipTipSetAndMessages(bs amt.Blocks, ts *types.TipSet, allbmsgs []*types.Mes
bmsgCids = append(bmsgCids, &c)
}
if msgc := len(bmsgCids) + len(smsgCids); msgc > build.BlockMessageLimit {
return nil, fmt.Errorf("block %q has too many messages (%d)", b.Cid(), msgc)
}
mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids)
if err != nil {
return nil, err
@ -536,7 +545,13 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
return xerrors.Errorf("no candidates")
}
wins := make(map[uint64]bool)
for _, t := range h.EPostProof.Candidates {
if wins[t.ChallengeIndex] {
return xerrors.Errorf("block had duplicate epost candidates")
}
wins[t.ChallengeIndex] = true
if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) {
return xerrors.Errorf("miner created a block but was not a winner")
}
@ -1173,3 +1188,7 @@ func (syncer *Syncer) State() []SyncerState {
}
return out
}
func (syncer *Syncer) MarkBad(blk cid.Cid) {
syncer.bad.Add(blk)
}

2
chain/sync_test.go
View File

@ -7,7 +7,7 @@ import (
"testing"
"time"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/peer"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
"github.com/stretchr/testify/require"

40
chain/types/bitfield.go
View File

@ -1,6 +1,7 @@
package types
import (
"errors"
"fmt"
"io"
@ -9,6 +10,8 @@ import (
"golang.org/x/xerrors"
)
var ErrBitFieldTooMany = errors.New("to many items in RLE")
type BitField struct {
rle rlepluslazy.RLE
@ -16,14 +19,23 @@ type BitField struct {
}
func NewBitField() BitField {
rle, err := rlepluslazy.FromBuf([]byte{})
bf, err := NewBitFieldFromBytes([]byte{})
if err != nil {
panic(err)
panic(fmt.Sprintf("creating empty rle: %+v", err))
}
return BitField{
rle: rle,
bits: make(map[uint64]struct{}),
return bf
}
func NewBitFieldFromBytes(rle []byte) (BitField, error) {
bf := BitField{}
rlep, err := rlepluslazy.FromBuf(rle)
if err != nil {
return BitField{}, xerrors.Errorf("could not decode rle+: %w", err)
}
bf.rle = rlep
bf.bits = make(map[uint64]struct{})
return bf, nil
}
func BitFieldFromSet(setBits []uint64) BitField {
@ -106,7 +118,14 @@ func (bf BitField) Count() (uint64, error) {
}
// All returns all set bits
func (bf BitField) All() ([]uint64, error) {
func (bf BitField) All(max uint64) ([]uint64, error) {
c, err := bf.Count()
if err != nil {
return nil, xerrors.Errorf("count errror: %w", err)
}
if c > max {
return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
}
runs, err := bf.sum()
if err != nil {
@ -121,7 +140,14 @@ func (bf BitField) All() ([]uint64, error) {
return res, nil
}
func (bf BitField) AllMap() (map[uint64]bool, error) {
func (bf BitField) AllMap(max uint64) (map[uint64]bool, error) {
c, err := bf.Count()
if err != nil {
return nil, xerrors.Errorf("count errror: %w", err)
}
if c > max {
return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
}
runs, err := bf.sum()
if err != nil {

13
chain/types/blockheader.go
View File

@ -5,6 +5,8 @@ import (
"context"
"math/big"
"github.com/filecoin-project/go-sectorbuilder"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/minio/sha256-simd"
@ -13,6 +15,7 @@ import (
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
)
@ -56,6 +59,8 @@ type BlockHeader struct {
Timestamp uint64
BlockSig *Signature
ForkSignaling uint64
}
func (b *BlockHeader) ToStorageBlock() (block.Block, error) {
@ -213,12 +218,8 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big
return lhs.Cmp(rhs) < 0
}
func ElectionPostChallengeCount(sectors uint64, faults int) uint64 {
if sectors == 0 {
return 0
}
// ceil(sectors / build.SectorChallengeRatioDiv)
return (sectors-uint64(faults)-1)/build.SectorChallengeRatioDiv + 1
func ElectionPostChallengeCount(sectors uint64, faults uint64) uint64 {
return sectorbuilder.ElectionPostChallengeCount(sectors, faults)
}
func (t *Ticket) Equals(ot *Ticket) bool {

21
chain/types/cbor_gen.go
View File

@ -5,7 +5,7 @@ import (
"io"
"math"
cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
@ -19,7 +19,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{140}); err != nil {
if _, err := w.Write([]byte{141}); err != nil {
return err
}
@ -94,6 +94,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
if err := t.BlockSig.MarshalCBOR(w); err != nil {
return err
}
// t.ForkSignaling (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil {
return err
}
return nil
}
@ -108,7 +113,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 12 {
if extra != 13 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -272,6 +277,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
}
}
// t.ForkSignaling (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ForkSignaling = uint64(extra)
return nil
}

2
chain/types/tipset.go
View File

@ -8,7 +8,7 @@ import (
"sort"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)

9
chain/vm/vm.go
View File

@ -10,7 +10,7 @@ import (
cid "github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
@ -259,7 +259,7 @@ func (bs *gasChargingBlocks) GetBlock(ctx context.Context, c cid.Cid) (block.Blo
}
blk, err := bs.under.GetBlock(ctx, c)
if err != nil {
return nil, err
return nil, aerrors.Escalate(err, "failed to get block from blockstore")
}
if err := bs.chargeGas(uint64(len(blk.RawData())) * gasGetPerByte); err != nil {
return nil, err
@ -272,7 +272,10 @@ func (bs *gasChargingBlocks) AddBlock(blk block.Block) error {
if err := bs.chargeGas(gasPutObj + uint64(len(blk.RawData()))*gasPutPerByte); err != nil {
return err
}
return bs.under.AddBlock(blk)
if err := bs.under.AddBlock(blk); err != nil {
return aerrors.Escalate(err, "failed to write data to disk")
}
return nil
}
func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Message, origin address.Address, usedGas types.BigInt) *VMContext {

2
chain/wallet/wallet.go
View File

@ -9,7 +9,7 @@ import (
bls "github.com/filecoin-project/filecoin-ffi"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/minio/blake2b-simd"
"golang.org/x/xerrors"

2
cli/cmd.go
View File

@ -9,7 +9,7 @@ import (
"strings"
"syscall"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr-net"

4
cli/params.go
View File

@ -5,6 +5,8 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"
)
var fetchParamCmd = &cli.Command{
@ -22,7 +24,7 @@ var fetchParamCmd = &cli.Command{
return err
}
sectorSize := uint64(sectorSizeInt)
err = paramfetch.GetParams(sectorSize)
err = paramfetch.GetParams(build.ParametersJson, sectorSize)
if err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}

72
cli/state.go
View File

@ -36,6 +36,7 @@ var stateCmd = &cli.Command{
stateReplaySetCmd,
stateSectorSizeCmd,
stateReadStateCmd,
stateListMessagesCmd,
},
}
@ -508,3 +509,74 @@ var stateReadStateCmd = &cli.Command{
return nil
},
}
var stateListMessagesCmd = &cli.Command{
Name: "list-messages",
Usage: "list messages on chain matching given criteria",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "to",
Usage: "return messages to a given address",
},
&cli.StringFlag{
Name: "from",
Usage: "return messages from a given address",
},
&cli.Uint64Flag{
Name: "toheight",
Usage: "don't look before given block height",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
var toa, froma address.Address
if tos := cctx.String("to"); tos != "" {
a, err := address.NewFromString(tos)
if err != nil {
return fmt.Errorf("given 'to' address %q was invalid: %w", tos, err)
}
toa = a
}
if froms := cctx.String("from"); froms != "" {
a, err := address.NewFromString(froms)
if err != nil {
return fmt.Errorf("given 'from' address %q was invalid: %w", froms, err)
}
froma = a
}
toh := cctx.Uint64("toheight")
ts, err := loadTipSet(ctx, cctx, api)
if err != nil {
return err
}
msgs, err := api.StateListMessages(ctx, &types.Message{To: toa, From: froma}, ts, toh)
if err != nil {
return err
}
for _, c := range msgs {
m, err := api.ChainGetMessage(ctx, c)
if err != nil {
return err
}
b, err := json.MarshalIndent(m, "", " ")
if err != nil {
return err
}
fmt.Println(string(b))
}
return nil
},
}

25
cli/sync.go
View File

@ -19,6 +19,7 @@ var syncCmd = &cli.Command{
Subcommands: []*cli.Command{
syncStatusCmd,
syncWaitCmd,
syncMarkBadCmd,
},
}
@ -90,6 +91,30 @@ var syncWaitCmd = &cli.Command{
},
}
var syncMarkBadCmd = &cli.Command{
Name: "mark-bad",
Usage: "Mark the given block as bad, will prevent syncing to a chain that contains it",
Action: func(cctx *cli.Context) error {
napi, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
if !cctx.Args().Present() {
return fmt.Errorf("must specify block cid to mark")
}
bcid, err := cid.Decode(cctx.Args().First())
if err != nil {
return fmt.Errorf("failed to decode input as a cid: %s", err)
}
return napi.SyncMarkBad(ctx, bcid)
},
}
func SyncWait(ctx context.Context, napi api.FullNode) error {
for {
state, err := napi.SyncState(ctx)

2
cli/wallet.go
View File

@ -41,7 +41,7 @@ var walletNew = &cli.Command{
t := cctx.Args().First()
if t == "" {
t = "bls"
t = "secp256k1"
}
nk, err := api.WalletNew(ctx, t)

10
cmd/lotus-bench/main.go
View File

@ -17,16 +17,16 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("lotus-bench")
@ -155,7 +155,7 @@ func main() {
}
}
if err := paramfetch.GetParams(sectorSize); err != nil {
if err := paramfetch.GetParams(build.ParametersJson, sectorSize); err != nil {
return xerrors.Errorf("getting params: %w", err)
}
sb, err := sectorbuilder.New(cfg, mds)
@ -188,7 +188,7 @@ func main() {
log.Info("Running replication...")
pieces := []sectorbuilder.PublicPieceInfo{pi}
pco, err := sb.SealPreCommit(i, ticket, pieces)
pco, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
@ -206,7 +206,7 @@ func main() {
}
log.Info("Generating PoRep for sector")
proof, err := sb.SealCommit(i, ticket, seed, pieces, pco)
proof, err := sb.SealCommit(context.TODO(), i, ticket, seed, pieces, pco)
if err != nil {
return err
}

2
cmd/lotus-chainwatch/main.go
View File

@ -5,7 +5,7 @@ import (
"net/http"
"os"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"

2
cmd/lotus-fountain/main.go
View File

@ -11,7 +11,7 @@ import (
rice "github.com/GeertJohan/go.rice"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
peer "github.com/libp2p/go-libp2p-peer"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"

5
cmd/lotus-seal-worker/main.go
View File

@ -5,13 +5,14 @@ import (
"github.com/mitchellh/go-homedir"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/node/repo"
manet "github.com/multiformats/go-multiaddr-net"
)
@ -19,7 +20,7 @@ import (
var log = logging.Logger("main")
func main() {
logging.SetLogLevel("*", "INFO")
lotuslog.SetupLogLevels()
log.Info("Starting lotus worker")

9
cmd/lotus-seal-worker/sub.go
View File

@ -5,10 +5,11 @@ import (
"net/http"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-sectorbuilder"
"golang.org/x/xerrors"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/build"
)
type worker struct {
@ -40,7 +41,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut
return err
}
if err := paramfetch.GetParams(ssize); err != nil {
if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil {
return xerrors.Errorf("get params: %w", err)
}
@ -102,7 +103,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
switch task.Type {
case sectorbuilder.WorkerPreCommit:
rspco, err := w.sb.SealPreCommit(task.SectorID, task.SealTicket, task.Pieces)
rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces)
if err != nil {
return errRes(xerrors.Errorf("precomitting: %w", err))
}
@ -116,7 +117,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
return errRes(xerrors.Errorf("pushing precommited data: %w", err))
}
case sectorbuilder.WorkerCommit:
proof, err := w.sb.SealCommit(task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
if err != nil {
return errRes(xerrors.Errorf("comitting: %w", err))
}

2
cmd/lotus-seal-worker/transfer.go
View File

@ -6,12 +6,12 @@ import (
"net/http"
"os"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
files "github.com/ipfs/go-ipfs-files"
"golang.org/x/xerrors"
"gopkg.in/cheggaaa/pb.v1"
"path/filepath"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/tarutil"
)

12
cmd/lotus-seed/main.go
View File

@ -7,8 +7,11 @@ import (
"encoding/json"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
badger "github.com/ipfs/go-ds-badger2"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
@ -17,7 +20,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("lotus-seed")
@ -196,7 +198,7 @@ var aggregateSectorDirsCmd = &cli.Command{
SectorSize: ssize,
Dir: destdir,
WorkerThreads: 2,
}, agmds)
}, namespace.Wrap(agmds, datastore.NewKey("/sectorbuilder")))
if err != nil {
return err
}
@ -257,7 +259,7 @@ var aggregateSectorDirsCmd = &cli.Command{
SectorSize: genm.SectorSize,
Dir: dir,
WorkerThreads: 2,
}, mds)
}, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
if err != nil {
return err
}

12
cmd/lotus-seed/seed/seed.go
View File

@ -10,8 +10,11 @@ import (
"os"
"path/filepath"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
badger "github.com/ipfs/go-ds-badger2"
logging "github.com/ipfs/go-log/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
@ -20,7 +23,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("preseal")
@ -43,7 +45,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
return nil, err
}
sb, err := sectorbuilder.New(cfg, mds)
sb, err := sectorbuilder.New(cfg, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
if err != nil {
return nil, err
}
@ -69,7 +71,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
fmt.Printf("sector-id: %d, piece info: %v", sid, pi)
pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
pco, err := sb.SealPreCommit(context.TODO(), sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
if err != nil {
return nil, xerrors.Errorf("commit: %w", err)
}

2
cmd/lotus-shed/main.go
View File

@ -3,7 +3,7 @@ package main
import (
"os"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"

2
cmd/lotus-shed/nonce-fix.go
View File

@ -54,7 +54,7 @@ var noncefix = &cli.Command{
return err
}
err = api.MpoolPush(ctx, smsg)
_, err = api.MpoolPush(ctx, smsg)
if err != nil {
return err
}

25
cmd/lotus-storage-miner/init.go
View File

@ -13,7 +13,8 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
"github.com/ipfs/go-datastore/namespace"
badger "github.com/ipfs/go-ds-badger2"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/mitchellh/go-homedir"
@ -21,15 +22,17 @@ import (
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/go-sectorbuilder"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
@ -93,7 +96,7 @@ var initCmd = &cli.Command{
}
log.Info("Checking proof parameters")
if err := paramfetch.GetParams(ssize); err != nil {
if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
@ -174,7 +177,7 @@ var initCmd = &cli.Command{
SectorSize: ssize,
WorkerThreads: 2,
Dir: pssb,
}, oldmds)
}, namespace.Wrap(oldmds, datastore.NewKey("/sectorbuilder")))
if err != nil {
return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err)
}
@ -183,7 +186,7 @@ var initCmd = &cli.Command{
SectorSize: ssize,
WorkerThreads: 2,
Dir: lr.Path(),
}, mds)
}, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
if err != nil {
return xerrors.Errorf("failed to open up sectorbuilder: %w", err)
}
@ -281,8 +284,13 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String())
proposal, err := utils.ToSharedStorageDealProposal(&sector.Deal)
if err != nil {
return err
}
deal := &deals.MinerDeal{
Proposal: sector.Deal,
MinerDeal: storagemarket.MinerDeal{
Proposal: *proposal,
ProposalCid: proposalCid,
State: lapi.DealComplete,
Ref: proposalCid, // TODO: This is super wrong, but there
@ -290,6 +298,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
// and this isn't even used after the deal enters Complete state
DealID: dealID,
SectorID: sector.SectorID,
},
}
b, err = cborutil.Dump(deal)

6
cmd/lotus-storage-miner/main.go
View File

@ -3,12 +3,13 @@ package main
import (
"os"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/tracing"
)
@ -18,8 +19,7 @@ var log = logging.Logger("main")
const FlagStorageRepo = "storagerepo"
func main() {
logging.SetLogLevel("*", "INFO")
logging.SetLogLevel("swarm", "WARN")
lotuslog.SetupLogLevels()
local := []*cli.Command{
runCmd,

2
cmd/lotus/daemon.go
View File

@ -68,7 +68,7 @@ var DaemonCmd = &cli.Command{
return xerrors.Errorf("repo init error: %w", err)
}
if err := paramfetch.GetParams(0); err != nil {
if err := paramfetch.GetParams(build.ParametersJson, 0); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}

9
cmd/lotus/main.go
View File

@ -4,23 +4,18 @@ import (
"context"
"os"
logging "github.com/ipfs/go-log"
"go.opencensus.io/trace"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/lotuslog"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/tracing"
)
func main() {
logging.SetLogLevel("*", "INFO")
logging.SetLogLevel("dht", "ERROR")
logging.SetLogLevel("swarm2", "WARN")
logging.SetLogLevel("bitswap", "WARN")
logging.SetLogLevel("pubsub", "WARN")
logging.SetLogLevel("connmgr", "WARN")
lotuslog.SetupLogLevels()
local := []*cli.Command{
DaemonCmd,

2
cmd/lotus/rpc.go
View File

@ -17,7 +17,7 @@ import (
"github.com/filecoin-project/lotus/node/impl"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr-net"
"golang.org/x/xerrors"

78
datatransfer/dagservice_impl.go
View File

@ -1,78 +0,0 @@
package datatransfer
import (
"context"
"reflect"
"github.com/ipfs/go-cid"
ipldformat "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-merkledag"
ipld "github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
)
// This file implements a VERY simple, incomplete version of the data transfer
// module that allows us to make the necessary insertions of data transfer
// functionality into the storage market
// It does not:
// -- actually validate requests
// -- support Push requests
// -- support multiple subscribers
// -- do any actual network coordination or use Graphsync
type dagserviceImpl struct {
dag ipldformat.DAGService
subscriber Subscriber
}
// NewDAGServiceDataTransfer returns a data transfer manager based on
// an IPLD DAGService
func NewDAGServiceDataTransfer(dag ipldformat.DAGService) Manager {
return &dagserviceImpl{dag, nil}
}
// RegisterVoucherType registers a validator for the given voucher type
// will error if voucher type does not implement voucher
// or if there is a voucher type registered with an identical identifier
func (impl *dagserviceImpl) RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error {
return nil
}
// open a data transfer that will send data to the recipient peer and
// transfer parts of the piece that match the selector
func (impl *dagserviceImpl) OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) {
return ChannelID{}, xerrors.Errorf("not implemented")
}
// open a data transfer that will request data from the sending peer and
// transfer parts of the piece that match the selector
func (impl *dagserviceImpl) OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) {
ctx, cancel := context.WithCancel(ctx)
go func() {
defer cancel()
err := merkledag.FetchGraph(ctx, baseCid, impl.dag)
var event Event
if err != nil {
event = Error
} else {
event = Complete
}
impl.subscriber(event, ChannelState{Channel: Channel{voucher: voucher}})
}()
return ChannelID{}, nil
}
// close an open channel (effectively a cancel)
func (impl *dagserviceImpl) CloseDataTransferChannel(x ChannelID) {}
// get status of a transfer
func (impl *dagserviceImpl) TransferChannelStatus(x ChannelID) Status { return ChannelNotFoundError }
// get notified when certain types of events happen
func (impl *dagserviceImpl) SubscribeToEvents(subscriber Subscriber) {
impl.subscriber = subscriber
}
// get all in progress transfers
func (impl *dagserviceImpl) InProgressChannels() map[ChannelID]ChannelState { return nil }

173
datatransfer/types.go
View File

@ -1,173 +0,0 @@
package datatransfer
import (
"context"
"reflect"
"github.com/ipfs/go-cid"
ipld "github.com/ipld/go-ipld-prime"
"github.com/libp2p/go-libp2p-core/peer"
)
// Voucher is used to validate
// a data transfer request against the underlying storage or retrieval deal
// that precipitated it. The only requirement is a voucher can read and write
// from bytes, and has a string identifier type
type Voucher interface {
// ToBytes converts the Voucher to raw bytes
ToBytes() ([]byte, error)
// FromBytes reads a Voucher from raw bytes
FromBytes([]byte) error
// Identifier is a unique string identifier for this voucher type
Identifier() string
}
// Status is the status of transfer for a given channel
type Status int
const (
// Ongoing means the data transfer is in progress
Ongoing Status = iota
// Completed means the data transfer is completed successfully
Completed
// Failed means the data transfer failed
Failed
// ChannelNotFoundError means the searched for data transfer does not exist
ChannelNotFoundError
)
// TransferID is an identifier for a data transfer, shared between
// request/responder and unique to the requester
type TransferID uint64
// ChannelID is a unique identifier for a channel, distinct by both the other
// party's peer ID + the transfer ID
type ChannelID struct {
to peer.ID
id TransferID
}
// Channel represents all the parameters for a single data transfer
type Channel struct {
// an identifier for this channel shared by request and responder, set by requester through protocol
transferID TransferID
// base CID for the piece being transferred
baseCid cid.Cid
// portion of Piece to return, specified by an IPLD selector
selector ipld.Node
// used to verify this channel
voucher Voucher
// the party that is sending the data (not who initiated the request)
sender peer.ID
// the party that is receiving the data (not who initiated the request)
recipient peer.ID
// expected amount of data to be transferred
totalSize uint64
}
// TransferID returns the transfer id for this channel
func (c Channel) TransferID() TransferID { return c.transferID }
// BaseCID returns the CID that is at the root of this data transfer
func (c Channel) BaseCID() cid.Cid { return c.baseCid }
// Selector returns the IPLD selector for this data transfer (represented as
// an IPLD node)
func (c Channel) Selector() ipld.Node { return c.selector }
// Voucher returns the voucher for this data transfer
func (c Channel) Voucher() Voucher { return c.voucher }
// Sender returns the peer id for the node that is sending data
func (c Channel) Sender() peer.ID { return c.sender }
// Recipient returns the peer id for the node that is receiving data
func (c Channel) Recipient() peer.ID { return c.recipient }
// TotalSize returns the total size for the data being transferred
func (c Channel) TotalSize() uint64 { return c.totalSize }
// ChannelState is immutable channel data plus mutable state
type ChannelState struct {
Channel
// total bytes sent from this node (0 if receiver)
sent uint64
// total bytes received by this node (0 if sender)
received uint64
}
// Sent returns the number of bytes sent
func (c ChannelState) Sent() uint64 { return c.sent }
// Received returns the number of bytes received
func (c ChannelState) Received() uint64 { return c.received }
// Event is a name for an event that occurs on a data transfer channel
type Event int
const (
// Open is an event occurs when a channel is first opened
Open Event = iota
// Progress is an event that gets emitted every time more data is transferred
Progress
// Error is an event that emits when an error occurs in a data transfer
Error
// Complete is emitted when a data transfer is complete
Complete
)
// Subscriber is a callback that is called when events are emitted
type Subscriber func(event Event, channelState ChannelState)
// RequestValidator is an interface implemented by the client of the
// data transfer module to validate requests
type RequestValidator interface {
// ValidatePush validates a push request received from the peer that will send data
ValidatePush(
sender peer.ID,
voucher Voucher,
baseCid cid.Cid,
selector ipld.Node) error
// ValidatePull validates a pull request received from the peer that will receive data
ValidatePull(
receiver peer.ID,
voucher Voucher,
baseCid cid.Cid,
selector ipld.Node) error
}
// Manager is the core interface presented by all implementations of
// of the data transfer sub system
type Manager interface {
// RegisterVoucherType registers a validator for the given voucher type
// will error if voucher type does not implement voucher
// or if there is a voucher type registered with an identical identifier
RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error
// open a data transfer that will send data to the recipient peer and
// open a data transfer that will send data to the recipient peer and
// transfer parts of the piece that match the selector
OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error)
// open a data transfer that will request data from the sending peer and
// transfer parts of the piece that match the selector
OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error)
// close an open channel (effectively a cancel)
CloseDataTransferChannel(x ChannelID)
// get status of a transfer
TransferChannelStatus(x ChannelID) Status
// get notified when certain types of events happen
SubscribeToEvents(subscriber Subscriber)
// get all in progress transfers
InProgressChannels() map[ChannelID]ChannelState
}

6
documentation/en/join-testnet.md
View File

@ -47,16 +47,16 @@ lotus sync wait
## Create your first address
Initialize a wallet using BLS signature formats:
Initialize a new wallet:
```sh
lotus wallet new bls
lotus wallet new
```
Here is an example of the response:
```sh
t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza
t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi
```
- Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds.

33
gen/main.go
View File

@ -9,11 +9,9 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/evtsm"
"github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storage"
)
@ -60,21 +58,6 @@ func main() {
os.Exit(1)
}
err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval",
retrieval.RetParams{},
retrieval.Query{},
retrieval.QueryResponse{},
retrieval.Unixfs0Offer{},
retrieval.DealProposal{},
retrieval.DealResponse{},
retrieval.Block{},
)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteTupleEncodersToFile("./chain/blocksync/cbor_gen.go", "blocksync",
blocksync.BlockSyncRequest{},
blocksync.BlockSyncResponse{},
@ -139,22 +122,6 @@ func main() {
os.Exit(1)
}
err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals",
deals.AskRequest{},
deals.AskResponse{},
deals.Proposal{},
deals.Response{},
deals.SignedResponse{},
deals.ClientDealProposal{},
deals.ClientDeal{},
deals.MinerDeal{},
deals.StorageDataTransferVoucher{},
)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage",
storage.SealTicket{},
storage.SealSeed{},

44
go.mod
View File

@ -9,18 +9,19 @@ require (
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/docker/go-units v0.4.0
github.com/fatih/color v1.7.0 // indirect
github.com/filecoin-project/chain-validation v0.0.3
github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed
github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590
github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42
github.com/filecoin-project/go-paramfetch v0.0.1
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/google/go-cmp v0.3.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
github.com/hashicorp/go-multierror v1.0.0
@ -29,15 +30,15 @@ require (
github.com/ipfs/go-bitswap v0.1.8
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
github.com/ipfs/go-car v0.0.2
github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1
github.com/ipfs/go-cid v0.0.4
github.com/ipfs/go-datastore v0.1.1
github.com/ipfs/go-ds-badger v0.0.7
github.com/ipfs/go-datastore v0.3.1
github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521
github.com/ipfs/go-filestore v0.0.2
github.com/ipfs/go-fs-lock v0.0.1
github.com/ipfs/go-graphsync v0.0.4
github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1
github.com/ipfs/go-ipfs-blockstore v0.1.1
github.com/ipfs/go-ipfs-blocksutil v0.0.1
github.com/ipfs/go-ipfs-chunker v0.0.1
github.com/ipfs/go-ipfs-ds-help v0.0.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
@ -46,11 +47,11 @@ require (
github.com/ipfs/go-ipfs-routing v0.1.0
github.com/ipfs/go-ipld-cbor v0.0.3
github.com/ipfs/go-ipld-format v0.0.2
github.com/ipfs/go-log v1.0.0
github.com/ipfs/go-log v1.0.1
github.com/ipfs/go-log/v2 v2.0.2
github.com/ipfs/go-merkledag v0.2.4
github.com/ipfs/go-path v0.0.7
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb
github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b
github.com/libp2p/go-libp2p v0.4.2
github.com/libp2p/go-libp2p-circuit v0.1.4
github.com/libp2p/go-libp2p-connmgr v0.1.0
@ -69,41 +70,36 @@ require (
github.com/libp2p/go-libp2p-tls v0.1.0
github.com/libp2p/go-libp2p-yamux v0.2.1
github.com/libp2p/go-maddr-filter v0.0.5
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mattn/go-sqlite3 v1.12.0
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/minio/sha256-simd v0.1.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mr-tron/base58 v1.1.3 // indirect
github.com/multiformats/go-base32 v0.0.3
github.com/multiformats/go-multiaddr v0.1.1
github.com/multiformats/go-multiaddr-dns v0.2.0
github.com/multiformats/go-multiaddr-net v0.1.1
github.com/multiformats/go-multihash v0.0.10
github.com/multiformats/go-varint v0.0.2
github.com/onsi/ginkgo v1.9.0 // indirect
github.com/onsi/gomega v1.6.0 // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/otiai10/copy v1.0.2
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
github.com/smartystreets/assertions v1.0.1 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/prometheus/common v0.2.0
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0
github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
go.opencensus.io v0.22.1
go.opencensus.io v0.22.2
go.uber.org/atomic v1.5.1 // indirect
go.uber.org/dig v1.7.0 // indirect
go.uber.org/fx v1.9.0
go.uber.org/goleak v0.10.0 // indirect
go.uber.org/multierr v1.4.0
go.uber.org/zap v1.10.0
go.uber.org/zap v1.13.0
go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
golang.org/x/sys v0.0.0-20200107162124-548cf772de50 // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5
golang.org/x/tools v0.0.0-20200108195415-316d2f248479 // indirect
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
gopkg.in/cheggaaa/pb.v1 v1.0.28
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8

151
go.sum
View File

@ -7,6 +7,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
@ -14,6 +16,8 @@ github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtix
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ=
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY=
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@ -23,7 +27,9 @@ github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw=
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@ -32,6 +38,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0=
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
@ -42,6 +49,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@ -51,7 +60,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY=
@ -66,6 +78,10 @@ github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzA
github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A=
github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04=
github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg=
github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@ -78,6 +94,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY=
github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8=
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M=
github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw=
@ -85,6 +103,28 @@ github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2j
github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cODZD2YzpTUtrOSxbEnWFcQHidNRZiRdvLxySjGvG/M=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY=
github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww=
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42 h1:7OW2AiWtwxwtYnC16CiXZ/idQ7w3W7W1hhX+N4YSAyQ=
github.com/filecoin-project/go-fil-markets v0.0.0-20200110170857-c200f161be42/go.mod h1:Q5fvJGMISyUIna19DpoqiqTas4L9RVD7w3Udv7uCrTo=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE=
github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254 h1:4IvlPad82JaNBtqh8fEAUIKWv8I3tguAJjGvUyHNZS4=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200109194458-9656ce473254/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
@ -114,9 +154,12 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
@ -138,6 +181,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA=
github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
@ -172,11 +217,12 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0=
github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ=
github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ=
github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ=
github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk=
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds=
github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@ -184,18 +230,24 @@ github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK
github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI=
github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU=
github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
github.com/ipfs/go-ds-badger v0.0.7 h1:NMyh88Q50HG6/S2YD58DLkq0c0/ZQPMbSojONH+PRf4=
github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 h1:GSygNXXIRXFVntMR3Yr30imf40H9cYDAw72Qb0za2nE=
github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521/go.mod h1:oDKWqmQbLyyml3mgMtHgB7qAf6cQOEsJPRLFzfwKQ5Q=
github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO3s=
github.com/ipfs/go-filestore v0.0.2/go.mod h1:KnZ41qJsCt2OX2mxZS0xsK3Psr0/oB93HMMssLujjVc=
github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0=
github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU=
github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ=
github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 h1:0xpCaaXvN8bPyws3ObiCn7G0KIfgbS9E132eL57dHx8=
github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1/go.mod h1:8yRx0xLUps1Xq8ZDnIwIVdQRp7JjA55gGvCiRHT91Vk=
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
@ -237,6 +289,16 @@ github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI=
github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA=
github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo=
github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo=
github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM=
github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM=
github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw=
github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto=
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
@ -256,8 +318,10 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b h1:ACSEK4f1SDQC+FJ4B4pqHFW14d7kEW2ufwXA/c7eLP0=
github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs=
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4=
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
@ -268,6 +332,8 @@ github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+
github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4=
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU=
github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A=
github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
@ -286,6 +352,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ=
@ -305,6 +372,8 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J
github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE=
github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0=
github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ=
github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s=
@ -312,6 +381,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A=
github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM=
github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8=
github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds=
github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0=
github.com/libp2p/go-libp2p v0.4.2 h1:p0cthB0jDNHO4gH2HzS8/nAMMXbfUlFHs0jwZ4U+F2g=
github.com/libp2p/go-libp2p v0.4.2/go.mod h1:MNmgUxUw5pMsdOzMlT0EE7oKjRasl+WyVwM0IBlpKgQ=
github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE=
@ -320,10 +391,12 @@ github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Z
github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE=
github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc=
github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg=
github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk=
github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU=
github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU=
github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8=
github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU=
@ -332,6 +405,8 @@ github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGc
github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I=
github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE=
github.com/libp2p/go-libp2p-core v0.0.9/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE=
github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI=
github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0=
github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8=
@ -377,6 +452,8 @@ github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMg
github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
github.com/libp2p/go-libp2p-peerstore v0.1.2/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M=
github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k=
github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
@ -399,11 +476,15 @@ github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu
github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ=
github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0=
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
github.com/libp2p/go-libp2p-secio v0.1.1/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng=
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA=
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8=
github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q=
github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU=
github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ=
github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU=
github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
@ -447,6 +528,7 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW
github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4=
github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw=
github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ=
github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw=
@ -476,6 +558,9 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@ -514,6 +599,7 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP
github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA=
github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
@ -527,14 +613,15 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM=
github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ=
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M=
github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/multiformats/go-varint v0.0.2 h1:6sUvyh2YHpJCb8RZ6eYzj6iJQ4+chWYmyIHxszqlPTA=
github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
@ -576,14 +663,21 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
@ -591,6 +685,7 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
@ -608,6 +703,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -615,7 +711,10 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg=
github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
@ -627,6 +726,14 @@ github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboa
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2biEZZXdBKt9HX7DN3bYGFUqljqqy0DqgnY=
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM=
github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI=
github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ=
github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@ -651,26 +758,32 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/dig v1.7.0 h1:E5/L92iQTNJTjfgJF2KgU+/JpMaiuvK2DHLBj0+kSZk=
go.uber.org/dig v1.7.0/go.mod h1:z+dSd2TP9Usi48jL8M3v63iSBVkiwtVyMKxMZYYauPg=
go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4=
go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E=
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
go4.org v0.0.0-20190313082347-94abd6928b1d h1:JkRdGP3zvTtTbabWSAC6n67ka30y7gOzWAah4XYJSfw=
go4.org v0.0.0-20190313082347-94abd6928b1d/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
@ -687,6 +800,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -697,7 +812,10 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -710,19 +828,23 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -741,11 +863,14 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA=
golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@ -763,8 +888,12 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE=
golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -781,7 +910,9 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

2
lib/auth/handler.go
View File

@ -5,7 +5,7 @@ import (
"net/http"
"strings"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apistruct"

3
lib/evtsm/sched.go
View File

@ -5,10 +5,9 @@ import (
"reflect"
"sync"
"github.com/filecoin-project/go-statestore"
"github.com/ipfs/go-datastore"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/statestore"
)
type StateHandler interface {

2
lib/jsonrpc/client.go
View File

@ -12,7 +12,7 @@ import (
"sync/atomic"
"github.com/gorilla/websocket"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
"go.opencensus.io/trace/propagation"
"golang.org/x/xerrors"

12
lib/lotuslog/levels.go Normal file
View File

@ -0,0 +1,12 @@
package lotuslog
import logging "github.com/ipfs/go-log/v2"
func SetupLogLevels() {
logging.SetLogLevel("*", "INFO")
logging.SetLogLevel("dht", "ERROR")
logging.SetLogLevel("swarm2", "WARN")
logging.SetLogLevel("bitswap", "WARN")
logging.SetLogLevel("pubsub", "WARN")
logging.SetLogLevel("connmgr", "WARN")
}

126
lib/sectorbuilder/files.go
View File

@ -1,126 +0,0 @@
package sectorbuilder
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"golang.org/x/xerrors"
)
func (sb *SectorBuilder) SectorName(sectorID uint64) string {
return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID)
}
func (sb *SectorBuilder) StagedSectorPath(sectorID uint64) string {
return filepath.Join(sb.filesystem.pathFor(dataStaging), sb.SectorName(sectorID))
}
func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string {
return filepath.Join(sb.filesystem.pathFor(dataUnsealed), sb.SectorName(sectorID))
}
func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
return os.OpenFile(sb.StagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
}
func (sb *SectorBuilder) SealedSectorPath(sectorID uint64) (string, error) {
path := filepath.Join(sb.filesystem.pathFor(dataSealed), sb.SectorName(sectorID))
return path, nil
}
func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) {
dir := filepath.Join(sb.filesystem.pathFor(dataCache), sb.SectorName(sectorID))
err := os.Mkdir(dir, 0755)
if os.IsExist(err) {
err = nil
}
return dir, err
}
func (sb *SectorBuilder) GetPath(typ string, sectorName string) (string, error) {
_, found := overheadMul[dataType(typ)]
if !found {
return "", xerrors.Errorf("unknown sector type: %s", typ)
}
return filepath.Join(sb.filesystem.pathFor(dataType(typ)), sectorName), nil
}
func (sb *SectorBuilder) TrimCache(sectorID uint64) error {
dir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return xerrors.Errorf("getting cache dir: %w", err)
}
files, err := ioutil.ReadDir(dir)
if err != nil {
return xerrors.Errorf("readdir: %w", err)
}
for _, file := range files {
if !strings.HasSuffix(file.Name(), ".dat") { // _aux probably
continue
}
if strings.HasSuffix(file.Name(), "-data-tree-r-last.dat") { // Want to keep
continue
}
if err := os.Remove(filepath.Join(dir, file.Name())); err != nil {
return xerrors.Errorf("rm %s: %w", file.Name(), err)
}
}
return nil
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var wait sync.Mutex
var werr error
wait.Lock()
go func() {
defer wait.Unlock()
var copied int64
copied, werr = io.CopyN(w, r, n)
if werr != nil {
log.Warnf("toReadableFile: copy error: %+v", werr)
}
err := w.Close()
if werr == nil && err != nil {
werr = err
log.Warnf("toReadableFile: close error: %+v", err)
return
}
if copied != n {
log.Warnf("copied different amount than expected: %d != %d", copied, n)
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
}
}()
return f, func() error {
wait.Lock()
return werr
}, nil
}

116
lib/sectorbuilder/fs.go
View File

@ -1,116 +0,0 @@
package sectorbuilder
import (
"github.com/filecoin-project/lotus/chain/types"
"golang.org/x/xerrors"
"os"
"path/filepath"
"sync"
"syscall"
)
type dataType string
const (
dataCache dataType = "cache"
dataStaging dataType = "staging"
dataSealed dataType = "sealed"
dataUnsealed dataType = "unsealed"
)
var overheadMul = map[dataType]uint64{ // * sectorSize
dataCache: 11, // TODO: check if true for 32G sectors
dataStaging: 1,
dataSealed: 1,
dataUnsealed: 1,
}
type fs struct {
path string
// in progress actions
reserved map[dataType]uint64
lk sync.Mutex
}
func openFs(dir string) *fs {
return &fs{
path: dir,
reserved: map[dataType]uint64{},
}
}
func (f *fs) init() error {
for _, dir := range []string{f.path,
f.pathFor(dataCache),
f.pathFor(dataStaging),
f.pathFor(dataSealed),
f.pathFor(dataUnsealed)} {
if err := os.Mkdir(dir, 0755); err != nil {
if os.IsExist(err) {
continue
}
return err
}
}
return nil
}
func (f *fs) pathFor(typ dataType) string {
_, found := overheadMul[typ]
if !found {
panic("unknown data path requested")
}
return filepath.Join(f.path, string(typ))
}
func (f *fs) reservedBytes() int64 {
var out int64
for _, r := range f.reserved {
out += int64(r)
}
return out
}
func (f *fs) reserve(typ dataType, size uint64) error {
f.lk.Lock()
defer f.lk.Unlock()
var fsstat syscall.Statfs_t
if err := syscall.Statfs(f.pathFor(typ), &fsstat); err != nil {
return err
}
fsavail := int64(fsstat.Bavail) * int64(fsstat.Bsize)
avail := fsavail - f.reservedBytes()
need := overheadMul[typ] * size
if int64(need) > avail {
return xerrors.Errorf("not enough space in '%s', need %s, available %s (fs: %s, reserved: %s)",
f.path,
types.NewInt(need).SizeStr(),
types.NewInt(uint64(avail)).SizeStr(),
types.NewInt(uint64(fsavail)).SizeStr(),
types.NewInt(uint64(f.reservedBytes())).SizeStr())
}
f.reserved[typ] += need
return nil
}
func (f *fs) free(typ dataType, sectorSize uint64) {
f.lk.Lock()
defer f.lk.Unlock()
f.reserved[typ] -= overheadMul[typ] * sectorSize
return
}

27
lib/sectorbuilder/mock.go
View File

@ -1,27 +0,0 @@
package sectorbuilder
import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) {
addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq")
if err != nil {
return nil, err
}
sb, err := New(&Config{
SectorSize: sectorSize,
Dir: dir,
WorkerThreads: 2,
Miner: addr,
}, ds)
if err != nil {
return nil, err
}
return sb, nil
}

174
lib/sectorbuilder/remote.go
View File

@ -1,174 +0,0 @@
package sectorbuilder
import (
"context"
"golang.org/x/xerrors"
)
type WorkerTaskType int
const (
WorkerPreCommit WorkerTaskType = iota
WorkerCommit
)
type WorkerTask struct {
Type WorkerTaskType
TaskID uint64
SectorID uint64
// preCommit
SealTicket SealTicket
Pieces []PublicPieceInfo
// commit
SealSeed SealSeed
Rspco RawSealPreCommitOutput
}
type workerCall struct {
task WorkerTask
ret chan SealRes
}
func (sb *SectorBuilder) AddWorker(ctx context.Context, cfg WorkerCfg) (<-chan WorkerTask, error) {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
taskCh := make(chan WorkerTask)
r := &remote{
sealTasks: taskCh,
busy: 0,
}
sb.remoteCtr++
sb.remotes[sb.remoteCtr] = r
go sb.remoteWorker(ctx, r, cfg)
return taskCh, nil
}
func (sb *SectorBuilder) returnTask(task workerCall) {
var ret chan workerCall
switch task.task.Type {
case WorkerPreCommit:
ret = sb.precommitTasks
case WorkerCommit:
ret = sb.commitTasks
default:
log.Error("unknown task type", task.task.Type)
}
go func() {
select {
case ret <- task:
case <-sb.stopping:
return
}
}()
}
func (sb *SectorBuilder) remoteWorker(ctx context.Context, r *remote, cfg WorkerCfg) {
defer log.Warn("Remote worker disconnected")
defer func() {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
for i, vr := range sb.remotes {
if vr == r {
delete(sb.remotes, i)
return
}
}
}()
precommits := sb.precommitTasks
if cfg.NoPreCommit {
precommits = nil
}
commits := sb.commitTasks
if cfg.NoCommit {
commits = nil
}
for {
select {
case task := <-commits:
sb.doTask(ctx, r, task)
case task := <-precommits:
sb.doTask(ctx, r, task)
case <-ctx.Done():
return
case <-sb.stopping:
return
}
r.lk.Lock()
r.busy = 0
r.lk.Unlock()
}
}
func (sb *SectorBuilder) doTask(ctx context.Context, r *remote, task workerCall) {
resCh := make(chan SealRes)
sb.remoteLk.Lock()
sb.remoteResults[task.task.TaskID] = resCh
sb.remoteLk.Unlock()
// send the task
select {
case r.sealTasks <- task.task:
case <-ctx.Done():
sb.returnTask(task)
return
}
r.lk.Lock()
r.busy = task.task.TaskID
r.lk.Unlock()
// wait for the result
select {
case res := <-resCh:
// send the result back to the caller
select {
case task.ret <- res:
case <-ctx.Done():
return
case <-sb.stopping:
return
}
case <-ctx.Done():
log.Warnf("context expired while waiting for task %d (sector %d): %s", task.task.TaskID, task.task.SectorID, ctx.Err())
return
case <-sb.stopping:
return
}
}
func (sb *SectorBuilder) TaskDone(ctx context.Context, task uint64, res SealRes) error {
sb.remoteLk.Lock()
rres, ok := sb.remoteResults[task]
if ok {
delete(sb.remoteResults, task)
}
sb.remoteLk.Unlock()
if !ok {
return xerrors.Errorf("task %d not found", task)
}
select {
case rres <- res:
return nil
case <-ctx.Done():
return ctx.Err()
}
}

84
lib/sectorbuilder/scrub.go
View File

@ -1,84 +0,0 @@
package sectorbuilder
import (
"io/ioutil"
"os"
"path/filepath"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"golang.org/x/xerrors"
)
type Fault struct {
SectorID uint64
Err error
}
func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault {
var faults []*Fault
for _, sector := range sectorSet.Values() {
err := sb.checkSector(sector.SectorID)
if err != nil {
faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err})
}
}
return faults
}
func (sb *SectorBuilder) checkSector(sectorID uint64) error {
cache, err := sb.sectorCacheDir(sectorID)
if err != nil {
return xerrors.Errorf("getting sector cache dir: %w", err)
}
if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil {
return err
}
if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil {
return err
}
// TODO: better validate this
if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be?
return err
}
dent, err := ioutil.ReadDir(cache)
if err != nil {
return xerrors.Errorf("reading cache dir %s", cache)
}
if len(dent) != 3 {
return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache)
}
sealed, err := sb.SealedSectorPath(sectorID)
if err != nil {
return xerrors.Errorf("getting sealed sector path: %w", err)
}
if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil {
return err
}
return nil
}
func assertFile(path string, minSz uint64, maxSz uint64) error {
st, err := os.Stat(path)
if err != nil {
return xerrors.Errorf("stat %s: %w", path, err)
}
if st.IsDir() {
return xerrors.Errorf("expected %s to be a regular file", path)
}
if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz {
return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size())
}
return nil
}

799
lib/sectorbuilder/sectorbuilder.go
View File

@ -1,799 +0,0 @@
package sectorbuilder
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
dcopy "github.com/otiai10/copy"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
const PoStReservedWorkers = 1
const PoRepProofPartitions = 10
var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
var log = logging.Logger("sectorbuilder")
type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo
type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo
type SealTicket = sectorbuilder.SealTicket
type SealSeed = sectorbuilder.SealSeed
type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput
type SealCommitOutput = sectorbuilder.SealCommitOutput
type PublicPieceInfo = sectorbuilder.PublicPieceInfo
type RawSealPreCommitOutput sectorbuilder.RawSealPreCommitOutput
type EPostCandidate = sectorbuilder.Candidate
const CommLen = sectorbuilder.CommitmentBytesLen
type WorkerCfg struct {
NoPreCommit bool
NoCommit bool
// TODO: 'cost' info, probably in terms of sealing + transfer speed
}
type SectorBuilder struct {
ds dtypes.MetadataDS
idLk sync.Mutex
ssize uint64
lastID uint64
Miner address.Address
unsealLk sync.Mutex
noCommit bool
noPreCommit bool
rateLimit chan struct{}
precommitTasks chan workerCall
commitTasks chan workerCall
taskCtr uint64
remoteLk sync.Mutex
remoteCtr int
remotes map[int]*remote
remoteResults map[uint64]chan<- SealRes
addPieceWait int32
preCommitWait int32
commitWait int32
unsealWait int32
fsLk sync.Mutex
filesystem *fs // TODO: multi-fs support
stopping chan struct{}
}
type JsonRSPCO struct {
CommD []byte
CommR []byte
}
func (rspco *RawSealPreCommitOutput) ToJson() JsonRSPCO {
return JsonRSPCO{
CommD: rspco.CommD[:],
CommR: rspco.CommR[:],
}
}
func (rspco *JsonRSPCO) rspco() RawSealPreCommitOutput {
var out RawSealPreCommitOutput
copy(out.CommD[:], rspco.CommD)
copy(out.CommR[:], rspco.CommR)
return out
}
type SealRes struct {
Err string
GoErr error `json:"-"`
Proof []byte
Rspco JsonRSPCO
}
type remote struct {
lk sync.Mutex
sealTasks chan<- WorkerTask
busy uint64 // only for metrics
}
type Config struct {
SectorSize uint64
Miner address.Address
WorkerThreads uint8
FallbackLastID uint64
NoCommit bool
NoPreCommit bool
Dir string
_ struct{} // guard against nameless init
}
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
if cfg.WorkerThreads < PoStReservedWorkers {
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers, cfg.WorkerThreads)
}
var lastUsedID uint64
b, err := ds.Get(lastSectorIdKey)
switch err {
case nil:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return nil, err
}
lastUsedID = uint64(i)
case datastore.ErrNotFound:
lastUsedID = cfg.FallbackLastID
default:
return nil, err
}
rlimit := cfg.WorkerThreads - PoStReservedWorkers
sealLocal := rlimit > 0
if rlimit == 0 {
rlimit = 1
}
sb := &SectorBuilder{
ds: ds,
ssize: cfg.SectorSize,
lastID: lastUsedID,
filesystem: openFs(cfg.Dir),
Miner: cfg.Miner,
noPreCommit: cfg.NoPreCommit || !sealLocal,
noCommit: cfg.NoCommit || !sealLocal,
rateLimit: make(chan struct{}, rlimit),
taskCtr: 1,
precommitTasks: make(chan workerCall),
commitTasks: make(chan workerCall),
remoteResults: map[uint64]chan<- SealRes{},
remotes: map[int]*remote{},
stopping: make(chan struct{}),
}
if err := sb.filesystem.init(); err != nil {
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
}
return sb, nil
}
func NewStandalone(cfg *Config) (*SectorBuilder, error) {
sb := &SectorBuilder{
ds: nil,
ssize: cfg.SectorSize,
Miner: cfg.Miner,
filesystem: openFs(cfg.Dir),
taskCtr: 1,
remotes: map[int]*remote{},
rateLimit: make(chan struct{}, cfg.WorkerThreads),
stopping: make(chan struct{}),
}
if err := sb.filesystem.init(); err != nil {
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
}
return sb, nil
}
func (sb *SectorBuilder) checkRateLimit() {
if cap(sb.rateLimit) == len(sb.rateLimit) {
log.Warn("rate-limiting local sectorbuilder call")
}
}
func (sb *SectorBuilder) RateLimit() func() {
sb.checkRateLimit()
sb.rateLimit <- struct{}{}
return func() {
<-sb.rateLimit
}
}
type WorkerStats struct {
LocalFree int
LocalReserved int
LocalTotal int
// todo: post in progress
RemotesTotal int
RemotesFree int
AddPieceWait int
PreCommitWait int
CommitWait int
UnsealWait int
}
func (sb *SectorBuilder) WorkerStats() WorkerStats {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
remoteFree := len(sb.remotes)
for _, r := range sb.remotes {
if r.busy > 0 {
remoteFree--
}
}
return WorkerStats{
LocalFree: cap(sb.rateLimit) - len(sb.rateLimit),
LocalReserved: PoStReservedWorkers,
LocalTotal: cap(sb.rateLimit) + PoStReservedWorkers,
RemotesTotal: len(sb.remotes),
RemotesFree: remoteFree,
AddPieceWait: int(atomic.LoadInt32(&sb.addPieceWait)),
PreCommitWait: int(atomic.LoadInt32(&sb.preCommitWait)),
CommitWait: int(atomic.LoadInt32(&sb.commitWait)),
UnsealWait: int(atomic.LoadInt32(&sb.unsealWait)),
}
}
func addressToProverID(a address.Address) [32]byte {
var proverId [32]byte
copy(proverId[:], a.Payload())
return proverId
}
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
sb.idLk.Lock()
defer sb.idLk.Unlock()
sb.lastID++
id := sb.lastID
err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
if err != nil {
return 0, err
}
return id, nil
}
func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) {
fs := sb.filesystem
if err := fs.reserve(dataStaging, sb.ssize); err != nil {
return PublicPieceInfo{}, err
}
defer fs.free(dataStaging, sb.ssize)
atomic.AddInt32(&sb.addPieceWait, 1)
ret := sb.RateLimit()
atomic.AddInt32(&sb.addPieceWait, -1)
defer ret()
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return PublicPieceInfo{}, err
}
stagedFile, err := sb.stagedSectorFile(sectorId)
if err != nil {
return PublicPieceInfo{}, err
}
_, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
if err != nil {
return PublicPieceInfo{}, err
}
if err := stagedFile.Close(); err != nil {
return PublicPieceInfo{}, err
}
if err := f.Close(); err != nil {
return PublicPieceInfo{}, err
}
return PublicPieceInfo{
Size: pieceSize,
CommP: commP,
}, werr()
}
func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
fs := sb.filesystem
if err := fs.reserve(dataUnsealed, sb.ssize); err != nil { // TODO: this needs to get smarter when we start supporting partial unseals
return nil, err
}
defer fs.free(dataUnsealed, sb.ssize)
atomic.AddInt32(&sb.unsealWait, 1)
// TODO: Don't wait if cached
ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker
defer ret()
atomic.AddInt32(&sb.unsealWait, -1)
sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel
defer sb.unsealLk.Unlock()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
sealedPath, err := sb.SealedSectorPath(sectorID)
if err != nil {
return nil, err
}
unsealedPath := sb.unsealedSectorPath(sectorID)
// TODO: GC for those
// (Probably configurable count of sectors to be kept unsealed, and just
// remove last used one (or use whatever other cache policy makes sense))
f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
var commd [CommLen]byte
copy(commd[:], commD)
var tkt [CommLen]byte
copy(tkt[:], ticket)
err = sectorbuilder.Unseal(sb.ssize,
PoRepProofPartitions,
cacheDir,
sealedPath,
unsealedPath,
sectorID,
addressToProverID(sb.Miner),
tkt,
commd)
if err != nil {
return nil, xerrors.Errorf("unseal failed: %w", err)
}
f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
return nil, err
}
}
if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
return nil, xerrors.Errorf("seek: %w", err)
}
lr := io.LimitReader(f, int64(size))
return &struct {
io.Reader
io.Closer
}{
Reader: lr,
Closer: f,
}, nil
}
func (sb *SectorBuilder) sealPreCommitRemote(call workerCall) (RawSealPreCommitOutput, error) {
atomic.AddInt32(&sb.preCommitWait, -1)
select {
case ret := <-call.ret:
var err error
if ret.Err != "" {
err = xerrors.New(ret.Err)
}
return ret.Rspco.rspco(), err
case <-sb.stopping:
return RawSealPreCommitOutput{}, xerrors.New("sectorbuilder stopped")
}
}
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
fs := sb.filesystem
if err := fs.reserve(dataCache, sb.ssize); err != nil {
return RawSealPreCommitOutput{}, err
}
defer fs.free(dataCache, sb.ssize)
if err := fs.reserve(dataSealed, sb.ssize); err != nil {
return RawSealPreCommitOutput{}, err
}
defer fs.free(dataSealed, sb.ssize)
call := workerCall{
task: WorkerTask{
Type: WorkerPreCommit,
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
SectorID: sectorID,
SealTicket: ticket,
Pieces: pieces,
},
ret: make(chan SealRes),
}
atomic.AddInt32(&sb.preCommitWait, 1)
select { // prefer remote
case sb.precommitTasks <- call:
return sb.sealPreCommitRemote(call)
default:
}
sb.checkRateLimit()
rl := sb.rateLimit
if sb.noPreCommit {
rl = make(chan struct{})
}
select { // use whichever is available
case sb.precommitTasks <- call:
return sb.sealPreCommitRemote(call)
case rl <- struct{}{}:
}
atomic.AddInt32(&sb.preCommitWait, -1)
// local
defer func() {
<-sb.rateLimit
}()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("getting cache dir: %w", err)
}
sealedPath, err := sb.SealedSectorPath(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("getting sealed sector path: %w", err)
}
e, err := os.OpenFile(sealedPath, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("ensuring sealed file exists: %w", err)
}
if err := e.Close(); err != nil {
return RawSealPreCommitOutput{}, err
}
var sum uint64
for _, piece := range pieces {
sum += piece.Size
}
ussize := UserBytesForSectorSize(sb.ssize)
if sum != ussize {
return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
}
stagedPath := sb.StagedSectorPath(sectorID)
rspco, err := sectorbuilder.SealPreCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
stagedPath,
sealedPath,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
pieces,
)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err)
}
return RawSealPreCommitOutput(rspco), nil
}
func (sb *SectorBuilder) sealCommitRemote(call workerCall) (proof []byte, err error) {
atomic.AddInt32(&sb.commitWait, -1)
select {
case ret := <-call.ret:
if ret.Err != "" {
err = xerrors.New(ret.Err)
}
return ret.Proof, err
case <-sb.stopping:
return nil, xerrors.New("sectorbuilder stopped")
}
}
func (sb *SectorBuilder) sealCommitLocal(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
atomic.AddInt32(&sb.commitWait, -1)
defer func() {
<-sb.rateLimit
}()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
proof, err = sectorbuilder.SealCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
seed.TicketBytes,
pieces,
sectorbuilder.RawSealPreCommitOutput(rspco),
)
if err != nil {
log.Warn("StandaloneSealCommit error: ", err)
log.Warnf("sid:%d tkt:%v seed:%v, ppi:%v rspco:%v", sectorID, ticket, seed, pieces, rspco)
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
call := workerCall{
task: WorkerTask{
Type: WorkerCommit,
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
SectorID: sectorID,
SealTicket: ticket,
Pieces: pieces,
SealSeed: seed,
Rspco: rspco,
},
ret: make(chan SealRes),
}
atomic.AddInt32(&sb.commitWait, 1)
select { // prefer remote
case sb.commitTasks <- call:
proof, err = sb.sealCommitRemote(call)
default:
sb.checkRateLimit()
rl := sb.rateLimit
if sb.noCommit {
rl = make(chan struct{})
}
select { // use whichever is available
case sb.commitTasks <- call:
proof, err = sb.sealCommitRemote(call)
case rl <- struct{}{}:
proof, err = sb.sealCommitLocal(sectorID, ticket, seed, pieces, rspco)
}
}
if err != nil {
return nil, xerrors.Errorf("commit: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) {
if len(challengeSeed) != CommLen {
return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen)
}
var cseed [CommLen]byte
copy(cseed[:], challengeSeed)
privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
}
func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
if err != nil {
return nil, err
}
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
}
func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) {
fmap := map[uint64]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []sectorbuilder.PrivateSectorInfo
for _, s := range sectorInfo.Values() {
if _, faulty := fmap[s.SectorID]; faulty {
continue
}
cachePath, err := sb.sectorCacheDir(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err)
}
sealedPath, err := sb.SealedSectorPath(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err)
}
out = append(out, sectorbuilder.PrivateSectorInfo{
SectorID: s.SectorID,
CommR: s.CommR,
CacheDirPath: cachePath,
SealedSectorPath: sealedPath,
})
}
return NewSortedPrivateSectorInfo(out), nil
}
func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
if err != nil {
return nil, nil, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
proverID := addressToProverID(sb.Miner)
candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, nil, err
}
proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates)
return candidates, proof, err
}
func (sb *SectorBuilder) Stop() {
close(sb.stopping)
}
func fallbackPostChallengeCount(sectors uint64, faults int) uint64 {
challengeCount := types.ElectionPostChallengeCount(sectors, faults)
if challengeCount > build.MaxFallbackPostChallengeCount {
return build.MaxFallbackPostChallengeCount
}
return challengeCount
}
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
if err := migrate(osb.filesystem.pathFor(dataCache), sb.filesystem.pathFor(dataCache), symlink); err != nil {
return err
}
if err := migrate(osb.filesystem.pathFor(dataStaging), sb.filesystem.pathFor(dataStaging), symlink); err != nil {
return err
}
if err := migrate(osb.filesystem.pathFor(dataSealed), sb.filesystem.pathFor(dataSealed), symlink); err != nil {
return err
}
val, err := osb.ds.Get(lastSectorIdKey)
if err != nil {
return err
}
if err := sb.ds.Put(lastSectorIdKey, val); err != nil {
return err
}
sb.lastID = osb.lastID
return nil
}
func (sb *SectorBuilder) SetLastSectorID(id uint64) error {
if err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))); err != nil {
return err
}
sb.lastID = id
return nil
}
func migrate(from, to string, symlink bool) error {
st, err := os.Stat(from)
if err != nil {
return err
}
if st.IsDir() {
return migrateDir(from, to, symlink)
}
return migrateFile(from, to, symlink)
}
func migrateDir(from, to string, symlink bool) error {
tost, err := os.Stat(to)
if err != nil {
if !os.IsNotExist(err) {
return err
}
if err := os.MkdirAll(to, 0755); err != nil {
return err
}
} else if !tost.IsDir() {
return xerrors.Errorf("target %q already exists and is a file (expected directory)")
}
dirents, err := ioutil.ReadDir(from)
if err != nil {
return err
}
for _, inf := range dirents {
n := inf.Name()
if inf.IsDir() {
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
return err
}
} else {
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
return err
}
}
}
return nil
}
func migrateFile(from, to string, symlink bool) error {
if symlink {
return os.Symlink(from, to)
}
return dcopy.Copy(from, to)
}

365
lib/sectorbuilder/sectorbuilder_test.go
View File

@ -1,365 +0,0 @@
package sectorbuilder_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"runtime"
"sync"
"testing"
"time"
ffi "github.com/filecoin-project/filecoin-ffi"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func init() {
logging.SetLogLevel("*", "INFO")
}
const sectorSize = 1024
type seal struct {
sid uint64
pco sectorbuilder.RawSealPreCommitOutput
ppi sectorbuilder.PublicPieceInfo
ticket sectorbuilder.SealTicket
}
func (s *seal) precommit(t *testing.T, sb *sectorbuilder.SectorBuilder, sid uint64, done func()) {
dlen := sectorbuilder.UserBytesForSectorSize(sectorSize)
var err error
r := io.LimitReader(rand.New(rand.NewSource(42+int64(sid))), int64(dlen))
s.ppi, err = sb.AddPiece(dlen, sid, r, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
s.ticket = sectorbuilder.SealTicket{
BlockHeight: 5,
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
}
s.pco, err = sb.SealPreCommit(sid, s.ticket, []sectorbuilder.PublicPieceInfo{s.ppi})
if err != nil {
t.Fatalf("%+v", err)
}
done()
}
func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()) {
seed := sectorbuilder.SealSeed{
BlockHeight: 15,
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
}
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifySeal(sectorSize, s.pco.CommR[:], s.pco.CommD[:], sb.Miner, s.ticket.TicketBytes[:], seed.TicketBytes[:], s.sid, proof)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("proof failed to validate")
}
done()
}
func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time {
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
ppi := make([]ffi.PublicSectorInfo, len(seals))
for i, s := range seals {
ppi[i] = ffi.PublicSectorInfo{
SectorID: s.sid,
CommR: s.pco.CommR,
}
}
ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi)
candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
genCandidates := time.Now()
if len(candndates) != 1 {
t.Fatal("expected 1 candidate")
}
postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("bad post")
}
return genCandidates
}
func TestSealAndVerify(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
si, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
s.commit(t, sb, func() {})
commit := time.Now()
genCandidiates := post(t, sb, s)
epost := time.Now()
// Restart sectorbuilder, re-run post
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
post(t, sb, s)
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("Commit: %s\n", commit.Sub(precommit).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealPoStNoCommit(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
si, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
// Restart sectorbuilder, re-run post
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
if err := sb.TrimCache(1); err != nil {
t.Fatal(err)
}
genCandidiates := post(t, sb, s)
epost := time.Now()
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealAndVerify2(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
var wg sync.WaitGroup
si1, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
si2, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s1 := seal{sid: si1}
s2 := seal{sid: si2}
wg.Add(2)
go s1.precommit(t, sb, 1, wg.Done)
time.Sleep(100 * time.Millisecond)
go s2.precommit(t, sb, 2, wg.Done)
wg.Wait()
wg.Add(2)
go s1.commit(t, sb, wg.Done)
go s2.commit(t, sb, wg.Done)
wg.Wait()
post(t, sb, s1, s2)
}
func TestAcquireID(t *testing.T) {
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire := func(expect uint64) {
id, err := sb.AcquireSectorId()
require.NoError(t, err)
assert.Equal(t, expect, id)
}
assertAcquire(1)
assertAcquire(2)
assertAcquire(3)
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire(4)
assertAcquire(5)
assertAcquire(6)
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}

75
lib/sectorbuilder/simple.go
View File

@ -1,75 +0,0 @@
package sectorbuilder
import (
"context"
"io"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"go.opencensus.io/trace"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
)
func (sb *SectorBuilder) SectorSize() uint64 {
return sb.ssize
}
var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
var commRa, commDa, ticketa, seeda [32]byte
copy(commRa[:], commR)
copy(commDa[:], commD)
copy(ticketa[:], ticket)
copy(seeda[:], seed)
proverIDa := addressToProverID(proverID)
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
}
func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo {
return sectorbuilder.NewSortedPrivateSectorInfo(sectors...)
}
func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo {
return sectorbuilder.NewSortedPublicSectorInfo(sectors...)
}
func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0)
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) {
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults)
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
var challengeSeeda [CommLen]byte
copy(challengeSeeda[:], challengeSeed)
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
prover := addressToProverID(proverID)
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover)
}
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
f, werr, err := toReadableFile(piece, int64(pieceSize))
if err != nil {
return [32]byte{}, err
}
commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
if err != nil {
return [32]byte{}, err
}
return commP, werr()
}
func GenerateDataCommitment(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([CommLen]byte, error) {
return sectorbuilder.GenerateDataCommitment(ssize, pieces)
}

93
lib/statestore/state.go
View File

@ -1,93 +0,0 @@
package statestore
import (
"bytes"
"reflect"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/ipfs/go-datastore"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
)
type StoredState struct {
ds datastore.Datastore
name datastore.Key
}
func (st *StoredState) End() error {
has, err := st.ds.Has(st.name)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", st.name)
}
if err := st.ds.Delete(st.name); err != nil {
return xerrors.Errorf("removing state from datastore: %w", err)
}
st.name = datastore.Key{}
st.ds = nil
return nil
}
func (st *StoredState) Get(out cbg.CBORUnmarshaler) error {
val, err := st.ds.Get(st.name)
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
return xerrors.Errorf("No state for %s: %w", st.name, err)
}
return err
}
return out.UnmarshalCBOR(bytes.NewReader(val))
}
// mutator func(*T) error
func (st *StoredState) Mutate(mutator interface{}) error {
return st.mutate(cborMutator(mutator))
}
func (st *StoredState) mutate(mutator func([]byte) ([]byte, error)) error {
has, err := st.ds.Has(st.name)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", st.name)
}
cur, err := st.ds.Get(st.name)
if err != nil {
return err
}
mutated, err := mutator(cur)
if err != nil {
return err
}
return st.ds.Put(st.name, mutated)
}
func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
rmut := reflect.ValueOf(mutator)
return func(in []byte) ([]byte, error) {
state := reflect.New(rmut.Type().In(0).Elem())
err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
if err != nil {
return nil, err
}
out := rmut.Call([]reflect.Value{state})
if err := out[0].Interface(); err != nil {
return nil, err.(error)
}
return cborutil.Dump(state.Interface())
}
}

96
lib/statestore/store.go
View File

@ -1,96 +0,0 @@
package statestore
import (
"bytes"
"fmt"
"reflect"
"github.com/filecoin-project/go-cbor-util"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
"go.uber.org/multierr"
"golang.org/x/xerrors"
)
type StateStore struct {
ds datastore.Datastore
}
func New(ds datastore.Datastore) *StateStore {
return &StateStore{ds: ds}
}
func ToKey(k interface{}) datastore.Key {
switch t := k.(type) {
case uint64:
return datastore.NewKey(fmt.Sprint(t))
case fmt.Stringer:
return datastore.NewKey(t.String())
default:
panic("unexpected key type")
}
}
func (st *StateStore) Begin(i interface{}, state interface{}) error {
k := ToKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if has {
return xerrors.Errorf("already tracking state for %v", i)
}
b, err := cborutil.Dump(state)
if err != nil {
return err
}
return st.ds.Put(k, b)
}
func (st *StateStore) Get(i interface{}) *StoredState {
return &StoredState{
ds: st.ds,
name: ToKey(i),
}
}
func (st *StateStore) Has(i interface{}) (bool, error) {
return st.ds.Has(ToKey(i))
}
// out: *[]T
func (st *StateStore) List(out interface{}) error {
res, err := st.ds.Query(query.Query{})
if err != nil {
return err
}
defer res.Close()
outT := reflect.TypeOf(out).Elem().Elem()
rout := reflect.ValueOf(out)
var errs error
for {
res, ok := res.NextSync()
if !ok {
break
}
if res.Error != nil {
return res.Error
}
elem := reflect.New(outT)
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
if err != nil {
errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
continue
}
rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
}
return nil
}

38
lib/statestore/store_test.go
View File

@ -1,38 +0,0 @@
package statestore
import (
"testing"
"github.com/ipfs/go-datastore"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/types"
)
func TestList(t *testing.T) {
ds := datastore.NewMapDatastore()
e, err := cborutil.Dump(types.NewInt(7))
if err != nil {
t.Fatal(err)
}
if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
t.Fatal(err)
}
st := &StateStore{ds: ds}
var out []types.BigInt
if err := st.List(&out); err != nil {
t.Fatal(err)
}
if len(out) != 1 {
t.Fatal("wrong len")
}
if out[0].Int64() != 7 {
t.Fatal("wrong data")
}
}

2
lib/tarutil/systar.go
View File

@ -8,7 +8,7 @@ import (
"os"
"path/filepath"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
)
var log = logging.Logger("tarutil")

49
markets/retrievaladapter/client.go Normal file
View File

@ -0,0 +1,49 @@
package retrievaladapter
import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/lotus/markets/utils"
payapi "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/paych"
)
type retrievalClientNode struct {
pmgr *paych.Manager
payapi payapi.PaychAPI
}
// NewRetrievalClientNode returns a new node adapter for a retrieval client that talks to the
// Lotus Node
func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClientNode {
return &retrievalClientNode{pmgr: pmgr, payapi: payapi}
}
// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist
// between a client and a miner and insures the client has the given amount of funds available in the channel
func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) {
paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable))
return paych, err
}
// Allocate late creates a lane within a payment channel so that calls to
// CreatePaymentVoucher will automatically make vouchers only for the difference
// in total
func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) {
return rcn.pmgr.AllocateLane(paymentChannel)
}
// CreatePaymentVoucher creates a new payment voucher in the given lane for a
// given payment channel so that all the payment vouchers in the lane add up
// to the given amount (so the payment voucher will be for the difference)
func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) {
voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane)
if err != nil {
return nil, err
}
return utils.ToSharedSignedVoucher(voucher)
}

48
markets/retrievaladapter/provider.go Normal file
View File

@ -0,0 +1,48 @@
package retrievaladapter
import (
"context"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
type retrievalProviderNode struct {
sectorBlocks *sectorblocks.SectorBlocks
full api.FullNode
}
// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the
// Lotus Node
func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode {
return &retrievalProviderNode{sectorBlocks, full}
}
func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) {
asCid, err := cid.Cast(pieceCid)
if err != nil {
return 0, err
}
return rpn.sectorBlocks.GetSize(asCid)
}
func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore {
return rpn.sectorBlocks.SealedBlockstore(approveUnseal)
}
func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) {
localVoucher, err := utils.FromSharedSignedVoucher(voucher)
if err != nil {
return retrievaltoken.FromInt(0), err
}
added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount))
return utils.ToSharedTokenAmount(added), err
}

341
markets/storageadapter/client.go Normal file
View File

@ -0,0 +1,341 @@
package storageadapter
// this file implements storagemarket.StorageClientNode
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/impl/full"
)
type ClientNodeAdapter struct {
full.StateAPI
full.ChainAPI
full.MpoolAPI
sm *stmgr.StateManager
cs *store.ChainStore
fm *market.FundMgr
ev *events.Events
}
type clientApi struct {
full.ChainAPI
full.StateAPI
}
func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode {
return &ClientNodeAdapter{
StateAPI: state,
ChainAPI: chain,
MpoolAPI: mpool,
sm: sm,
cs: cs,
fm: fm,
ev: events.NewEvents(context.TODO(), &clientApi{chain, state}),
}
}
func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storagemarket.StorageProviderInfo, error) {
ts, err := n.ChainHead(ctx)
if err != nil {
return nil, err
}
addresses, err := n.StateListMiners(ctx, ts)
if err != nil {
return nil, err
}
var out []*storagemarket.StorageProviderInfo
for _, addr := range addresses {
workerAddr, err := n.StateMinerWorker(ctx, addr, ts)
if err != nil {
return nil, err
}
sectorSize, err := n.StateMinerSectorSize(ctx, addr, ts)
if err != nil {
return nil, err
}
peerId, err := n.StateMinerPeerID(ctx, addr, ts)
if err != nil {
return nil, err
}
storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId)
out = append(out, &storageProviderInfo)
}
return out, nil
}
func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) {
allDeals, err := n.StateMarketDeals(ctx, nil)
if err != nil {
return nil, err
}
var out []storagemarket.StorageDeal
for _, deal := range allDeals {
storageDeal := utils.FromOnChainDeal(deal)
if storageDeal.Client == addr {
out = append(out, storageDeal)
}
}
return out, nil
}
func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) {
return n.ChainHead(ctx)
}
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
// (Provider Node API)
smsg, err := n.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: addr,
Value: utils.FromSharedTokenAmount(amount),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance,
})
if err != nil {
return err
}
r, err := n.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return err
}
if r.Receipt.ExitCode != 0 {
return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)
}
return nil
}
func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount))
}
func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
bal, err := n.StateMarketBalance(ctx, addr, nil)
if err != nil {
return storagemarket.Balance{}, err
}
return utils.ToSharedBalance(bal), nil
}
// ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal
// returns the Deal id if there is no error
func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (uint64, error) {
log.Infow("DEAL ACCEPTED!")
pubmsg, err := c.cs.GetMessage(*deal.PublishMessage)
if err != nil {
return 0, xerrors.Errorf("getting deal pubsish message: %w", err)
}
pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
if err != nil {
return 0, xerrors.Errorf("getting miner worker failed: %w", err)
}
if pubmsg.From != pw {
return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
}
if pubmsg.To != actors.StorageMarketAddress {
return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
}
if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
}
var params actors.PublishStorageDealsParams
if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
return 0, err
}
dealIdx := -1
for i, storageDeal := range params.Deals {
// TODO: make it less hacky
sd := storageDeal
eq, err := cborutil.Equals(&deal.Proposal, &sd)
if err != nil {
return 0, err
}
if eq {
dealIdx = i
break
}
}
if dealIdx == -1 {
return 0, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage)
}
// TODO: timeout
_, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage)
if err != nil {
return 0, xerrors.Errorf("waiting for deal publish message: %w", err)
}
if ret.ExitCode != 0 {
return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
}
var res actors.PublishStorageDealResponse
if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
return 0, err
}
return res.DealIDs[dealIdx], nil
}
func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb storagemarket.DealSectorCommittedCallback) error {
checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts)
if err != nil {
// TODO: This may be fine for some errors
return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
if sd.ActivationEpoch > 0 {
cb(nil)
return true, false, nil
}
return false, true, nil
}
called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) {
defer func() {
if err != nil {
cb(xerrors.Errorf("handling applied event: %w", err))
}
}()
if msg == nil {
log.Error("timed out waiting for deal activation... what now?")
return false, nil
}
sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts)
if err != nil {
return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
if sd.ActivationEpoch == 0 {
return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", dealId, ts.ParentState(), ts.Height())
}
log.Infof("Storage deal %d activated at epoch %d", dealId, sd.ActivationEpoch)
cb(nil)
return false, nil
}
revert := func(ctx context.Context, ts *types.TipSet) error {
log.Warn("deal activation reverted; TODO: actually handle this!")
// TODO: Just go back to DealSealing?
return nil
}
matchEvent := func(msg *types.Message) (bool, error) {
if msg.To != provider {
return false, nil
}
if msg.Method != actors.MAMethods.ProveCommitSector {
return false, nil
}
var params actors.SectorProveCommitInfo
if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
return false, err
}
var found bool
for _, dealID := range params.DealIDs {
if dealID == dealId {
found = true
break
}
}
return found, nil
}
if err := c.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
return xerrors.Errorf("failed to set up called handler")
}
return nil
}
func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error {
localProposal, err := utils.FromSharedStorageDealProposal(proposal)
if err != nil {
return err
}
err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal)
if err != nil {
return err
}
signature, err := utils.ToSharedSignature(localProposal.ProposerSignature)
if err != nil {
return err
}
proposal.ProposerSignature = signature
return nil
}
func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) {
addr, err := n.Wallet.GetDefault()
return addr, err
}
func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error {
tss := n.cs.GetHeaviestTipSet().ParentState()
w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner)
if err != nil {
return xerrors.Errorf("failed to get worker for miner in ask", err)
}
sigb, err := cborutil.Dump(ask.Ask)
if err != nil {
return xerrors.Errorf("failed to re-serialize ask")
}
return ask.Signature.Verify(w, sigb)
}
var _ storagemarket.StorageClientNode = &ClientNodeAdapter{}

209
markets/storageadapter/provider.go Normal file
View File

@ -0,0 +1,209 @@
package storageadapter
// this file implements storagemarket.StorageProviderNode
import (
"bytes"
"context"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
unixfile "github.com/ipfs/go-unixfs/file"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/markets/utils"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
var log = logging.Logger("provideradapter")
type ProviderNodeAdapter struct {
api.FullNode
// this goes away with the data transfer module
dag dtypes.StagingDAG
secb *sectorblocks.SectorBlocks
}
func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode {
return &ProviderNodeAdapter{
FullNode: full,
dag: dag,
secb: secb,
}
}
func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) {
log.Info("publishing deal")
worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil)
if err != nil {
return 0, cid.Undef, err
}
localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal)
if err != nil {
return 0, cid.Undef, err
}
params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
Deals: []actors.StorageDealProposal{*localProposal},
})
if err != nil {
return 0, cid.Undef, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
}
// TODO: We may want this to happen after fetching data
smsg, err := n.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: worker,
Value: types.NewInt(0),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.PublishStorageDeals,
Params: params,
})
if err != nil {
return 0, cid.Undef, err
}
r, err := n.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return 0, cid.Undef, err
}
if r.Receipt.ExitCode != 0 {
return 0, cid.Undef, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)
}
var resp actors.PublishStorageDealResponse
if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
return 0, cid.Undef, err
}
if len(resp.DealIDs) != 1 {
return 0, cid.Undef, xerrors.Errorf("got unexpected number of DealIDs from")
}
return storagemarket.DealID(resp.DealIDs[0]), smsg.Cid(), nil
}
func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, piecePath string) (uint64, error) {
root, err := n.dag.Get(ctx, deal.Ref)
if err != nil {
return 0, xerrors.Errorf("failed to get file root for deal: %s", err)
}
// TODO: abstract this away into ReadSizeCloser + implement different modes
node, err := unixfile.NewUnixfsFile(ctx, n.dag, root)
if err != nil {
return 0, xerrors.Errorf("cannot open unixfs file: %s", err)
}
uf, ok := node.(sectorblocks.UnixfsReader)
if !ok {
// we probably got directory, unsupported for now
return 0, xerrors.Errorf("unsupported unixfs file type")
}
// TODO: uf.Size() is user input, not trusted
// This won't be useful / here after we migrate to putting CARs into sectors
size, err := uf.Size()
if err != nil {
return 0, xerrors.Errorf("getting unixfs file size: %w", err)
}
if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
return 0, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
}
sectorID, err := n.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
if err != nil {
return 0, xerrors.Errorf("AddPiece failed: %s", err)
}
log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
return sectorID, nil
}
func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) {
allDeals, err := n.StateMarketDeals(ctx, nil)
if err != nil {
return nil, err
}
var out []storagemarket.StorageDeal
for _, deal := range allDeals {
sharedDeal := utils.FromOnChainDeal(deal)
if sharedDeal.Provider == addr {
out = append(out, sharedDeal)
}
}
return out, nil
}
func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) {
addr, err := n.StateMinerWorker(ctx, miner, nil)
return addr, err
}
func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) {
localSignature, err := n.WalletSign(ctx, signer, b)
if err != nil {
return nil, err
}
return utils.ToSharedSignature(localSignature)
}
func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error {
return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt))
}
func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) {
return n.ChainHead(ctx)
}
// Adds funds with the StorageMinerActor for a storage participant. Used by both providers and clients.
func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
// (Provider Node API)
smsg, err := n.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: addr,
Value: utils.FromSharedTokenAmount(amount),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance,
})
if err != nil {
return err
}
r, err := n.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return err
}
if r.Receipt.ExitCode != 0 {
return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)
}
return nil
}
func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
bal, err := n.StateMarketBalance(ctx, addr, nil)
if err != nil {
return storagemarket.Balance{}, err
}
return utils.ToSharedBalance(bal), nil
}
var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{}

174
markets/utils/converters.go Normal file
View File

@ -0,0 +1,174 @@
package utils
import (
"bytes"
peer "github.com/libp2p/go-libp2p-peer"
"github.com/filecoin-project/go-address"
sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
"github.com/filecoin-project/go-fil-markets/storagemarket"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
)
func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt {
return types.BigInt{Int: in.Int}
}
func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount {
return sharedamount.TokenAmount{Int: in.Int}
}
func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.SignedVoucher
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.SignedVoucher
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.Signature
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.Signature
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out sharedtypes.SignedStorageAsk
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) {
var encoded bytes.Buffer
err := in.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out types.SignedStorageAsk
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo {
return storagemarket.StorageProviderInfo{
Address: address,
Worker: miner,
SectorSize: sectorSize,
PeerID: peer,
}
}
func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal {
return storagemarket.StorageDeal{
PieceRef: deal.PieceRef,
PieceSize: deal.PieceSize,
Client: deal.Client,
Provider: deal.Provider,
StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch),
StorageCollateral: ToSharedTokenAmount(deal.StorageCollateral),
ActivationEpoch: deal.ActivationEpoch,
}
}
func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal {
return actors.OnChainDeal{
PieceRef: deal.PieceRef,
PieceSize: deal.PieceSize,
Client: deal.Client,
Provider: deal.Provider,
StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch),
StorageCollateral: FromSharedTokenAmount(deal.StorageCollateral),
ActivationEpoch: deal.ActivationEpoch,
}
}
func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance {
return storagemarket.Balance{
Locked: ToSharedTokenAmount(balance.Locked),
Available: ToSharedTokenAmount(balance.Available),
}
}
func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) {
var encoded bytes.Buffer
err := proposal.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out storagemarket.StorageDealProposal
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}
func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) {
var encoded bytes.Buffer
err := proposal.MarshalCBOR(&encoded)
if err != nil {
return nil, err
}
var out actors.StorageDealProposal
err = out.UnmarshalCBOR(&encoded)
if err != nil {
return nil, err
}
return &out, nil
}

14
miner/miner.go
View File

@ -12,13 +12,11 @@ import (
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
)
const MaxMessagesPerBlock = 4000
var log = logging.Logger("miner")
type waitFunc func(ctx context.Context, baseTime uint64) error
@ -363,6 +361,11 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type
return nil, xerrors.Errorf("message filtering failed: %w", err)
}
if len(msgs) > build.BlockMessageLimit {
log.Error("selectMessages returned too many messages: ", len(msgs))
msgs = msgs[:build.BlockMessageLimit]
}
uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(base.nullRounds+1))
nheight := base.ts.Height() + base.nullRounds + 1
@ -383,12 +386,13 @@ func countFrom(msgs []*types.SignedMessage, from address.Address) (out int) {
}
func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs []*types.SignedMessage) ([]*types.SignedMessage, error) {
out := make([]*types.SignedMessage, 0, len(msgs))
out := make([]*types.SignedMessage, 0, build.BlockMessageLimit)
inclNonces := make(map[address.Address]uint64)
inclBalances := make(map[address.Address]types.BigInt)
inclCount := make(map[address.Address]int)
for _, msg := range msgs {
if msg.Message.To == address.Undef {
log.Warnf("message in mempool had bad 'To' address")
continue
@ -426,7 +430,7 @@ func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs
inclCount[from]++
out = append(out, msg)
if len(out) >= MaxMessagesPerBlock {
if len(out) >= build.BlockMessageLimit {
break
}
}

34
node/builder.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"time"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
blockstore "github.com/ipfs/go-ipfs-blockstore"
ci "github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/host"
@ -18,10 +19,13 @@ import (
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
"github.com/filecoin-project/go-fil-markets/storagemarket"
deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/messagepool"
@ -30,7 +34,7 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello"
@ -43,8 +47,6 @@ import (
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/peermgr"
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -220,14 +222,15 @@ func Online() Option {
Override(RunPeerMgrKey, modules.RunPeerMgr),
Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
Override(new(*discovery.Local), discovery.NewLocal),
Override(new(discovery.PeerResolver), modules.RetrievalResolver),
Override(new(*discovery.Local), modules.NewLocalDiscovery),
Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver),
Override(new(*retrieval.Client), retrieval.NewClient),
Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient),
Override(new(dtypes.ClientDealStore), modules.NewClientDealStore),
Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer),
Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator),
Override(new(*deals.Client), deals.NewClient),
Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator),
Override(new(storagemarket.StorageClient), modules.StorageClient),
Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
Override(RegisterClientValidatorKey, modules.RegisterClientValidator),
Override(RunDealClientKey, modules.RunDealClient),
@ -238,18 +241,20 @@ func Online() Option {
// Storage miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder),
Override(new(storage.SectorBuilder), modules.SectorBuilder),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(storage.TicketFn), modules.SealTicketGen),
Override(new(*storage.Miner), modules.StorageMiner),
Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
Override(new(dtypes.StagingDAG), modules.StagingDAG),
Override(new(*retrieval.Miner), retrieval.NewMiner),
Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync),
Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider),
Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore),
Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer),
Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator),
Override(new(*deals.Provider), deals.NewProvider),
Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator),
Override(new(storagemarket.StorageProvider), modules.StorageProvider),
Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter),
Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator),
Override(HandleRetrievalKey, modules.HandleRetrieval),
Override(GetParamsKey, modules.GetParams),
@ -372,6 +377,7 @@ func Repo(r repo.Repo) Option {
Override(new(dtypes.ClientFilestore), modules.ClientFstore),
Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
Override(new(dtypes.ClientDAG), modules.ClientDAG),
Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync),
Override(new(ci.PrivKey), lp2p.PrivKey),
Override(new(ci.PubKey), ci.PrivKey.GetPublic),

2
node/fxlog.go
View File

@ -1,7 +1,7 @@
package node
import (
logging "github.com/ipfs/go-log"
logging "github.com/ipfs/go-log/v2"
"go.uber.org/fx"
)

Some files were not shown because too many files have changed in this diff Show More