cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/testnet2

Browse Source
This commit is contained in:
Łukasz Magiera 2020-01-07 21:45:30 +01:00
commit 22e9815928
55 changed files with 72 additions and 2444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.circleci/config.yml
View File

@ -37,14 +37,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}'
- 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-'
- 'v20-1k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params --proving-params 1024
- save_cache:
name: Save parameters cache
key: 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}'
key: 'v20-1k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/

2
api/api_storage.go
View File

@ -4,7 +4,7 @@ import (
"context"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/go-sectorbuilder"
)
// alias because cbor-gen doesn't like non-alias types

2
api/apistruct/struct.go
View File

@ -3,7 +3,7 @@ package apistruct
import (
"context"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/network"

8
build/genesis.go
View File

@ -1,6 +1,12 @@
package build
import rice "github.com/GeertJohan/go.rice"
import (
rice "github.com/GeertJohan/go.rice"
logging "github.com/ipfs/go-log"
)
// moved from now-defunct build/paramfetch.go
var log = logging.Logger("build")
func MaybeGenesis() []byte {
builtinGen, err := rice.FindBox("genesis")

189
build/paramfetch.go
View File

@ -1,189 +0,0 @@
package build
import (
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
rice "github.com/GeertJohan/go.rice"
logging "github.com/ipfs/go-log"
"github.com/minio/blake2b-simd"
"go.uber.org/multierr"
"golang.org/x/xerrors"
pb "gopkg.in/cheggaaa/pb.v1"
)
var log = logging.Logger("build")
//const gateway = "http://198.211.99.118/ipfs/"
const gateway = "https://ipfs.io/ipfs/"
const paramdir = "/var/tmp/filecoin-proof-parameters"
const dirEnv = "FIL_PROOFS_PARAMETER_CACHE"
type paramFile struct {
Cid string `json:"cid"`
Digest string `json:"digest"`
SectorSize uint64 `json:"sector_size"`
}
type fetch struct {
wg sync.WaitGroup
fetchLk sync.Mutex
errs []error
}
func getParamDir() string {
if os.Getenv(dirEnv) == "" {
return paramdir
}
return os.Getenv(dirEnv)
}
func GetParams(storageSize uint64) error {
if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) {
return err
}
var params map[string]paramFile
paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json")
if err := json.Unmarshal(paramBytes, &params); err != nil {
return err
}
ft := &fetch{}
for name, info := range params {
if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") {
continue
}
ft.maybeFetchAsync(name, info)
}
return ft.wait()
}
func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
ft.wg.Add(1)
go func() {
defer ft.wg.Done()
path := filepath.Join(getParamDir(), name)
err := ft.checkFile(path, info)
if !os.IsNotExist(err) && err != nil {
log.Warn(err)
}
if err == nil {
return
}
ft.fetchLk.Lock()
defer ft.fetchLk.Unlock()
if err := doFetch(path, info); err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err))
return
}
err = ft.checkFile(path, info)
if err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err))
err := os.Remove(path)
if err != nil {
ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err))
}
}
}()
}
func (ft *fetch) checkFile(path string, info paramFile) error {
if os.Getenv("TRUST_PARAMS") == "1" {
log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
return nil
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
h := blake2b.New512()
if _, err := io.Copy(h, f); err != nil {
return err
}
sum := h.Sum(nil)
strSum := hex.EncodeToString(sum[:16])
if strSum == info.Digest {
log.Infof("Parameter file %s is ok", path)
return nil
}
return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
}
func (ft *fetch) wait() error {
ft.wg.Wait()
return multierr.Combine(ft.errs...)
}
func doFetch(out string, info paramFile) error {
gw := os.Getenv("IPFS_GATEWAY")
if gw == "" {
gw = gateway
}
log.Infof("Fetching %s from %s", out, gw)
outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
return err
}
defer outf.Close()
fStat, err := outf.Stat()
if err != nil {
return err
}
header := http.Header{}
header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-")
url, err := url.Parse(gw + info.Cid)
if err != nil {
return err
}
log.Infof("GET %s", url)
req := http.Request{
Method: "GET",
URL: url,
Header: header,
Close: true,
}
resp, err := http.DefaultClient.Do(&req)
if err != nil {
return err
}
defer resp.Body.Close()
bar := pb.New64(resp.ContentLength)
bar.Units = pb.U_BYTES
bar.ShowSpeed = true
bar.Start()
_, err = io.Copy(outf, bar.NewProxyReader(resp.Body))
bar.Finish()
return err
}

5
build/params_shared.go
View File

@ -56,11 +56,6 @@ const SealRandomnessLookback = Finality
// Epochs
const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
// 1 / n
const SectorChallengeRatioDiv = 25
const MaxFallbackPostChallengeCount = 10
// /////
// Mining

20
build/proof-params/mkparamfetch.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
function b64() {
f="$1"
case `uname` in
Darwin)
base64 -i "$f"
;;
Linux)
base64 "$f" -w 0
;;
esac
printf "unsupported system" 1>&2
exit 1
}
sed "s/{{PARAMSJSON}}/$(b64 build/proof-params/parameters.json)/g" build/proof-params/paramfetch.sh.template > ./build/paramfetch.sh
chmod +x ./build/paramfetch.sh

103
build/proof-params/parameters.json
View File

@ -1,103 +0,0 @@
{
"v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": {
"cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9",
"digest": "39a9edec3355516674f0d12b926be493",
"sector_size": 34359738368
},
"v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": {
"cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX",
"digest": "0227ae7df4f2affe529ebafbbc7540ee",
"sector_size": 34359738368
},
"v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": {
"cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv",
"digest": "991041a354b12c280542741f58c7f2ca",
"sector_size": 1024
},
"v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": {
"cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU",
"digest": "91fac550e1f9bccab213830bb0c85bd6",
"sector_size": 1024
},
"v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": {
"cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4",
"digest": "6322eacd2773163ddd51f9ca7d645fc4",
"sector_size": 1073741824
},
"v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": {
"cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf",
"digest": "0df62745fceac922e3e70847cfc70b52",
"sector_size": 1073741824
},
"v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": {
"cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9",
"digest": "de9d71e672f286706a1673bd57abdaac",
"sector_size": 16777216
},
"v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": {
"cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu",
"digest": "7f796d3a0f13499181e44b5eee0cc744",
"sector_size": 16777216
},
"v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": {
"cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN",
"digest": "151ae0ae183fc141e8c2bebc28e5cc10",
"sector_size": 268435456
},
"v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": {
"cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji",
"digest": "95eb89588e9d1832aca044c3a13178af",
"sector_size": 268435456
},
"v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": {
"cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX",
"digest": "fcd50e2e08a8560a6bb3418e883567ed",
"sector_size": 268435456
},
"v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": {
"cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y",
"digest": "872e244d86499fd659082e3bcf3f13e7",
"sector_size": 268435456
},
"v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": {
"cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN",
"digest": "0a338b941c5f17946340de5fc95cab30",
"sector_size": 34359738368
},
"v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": {
"cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF",
"digest": "c752e070a6b7aa8b79aa661a6b600b55",
"sector_size": 34359738368
},
"v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": {
"cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186",
"digest": "efa407fd09202dffd15799a8518e73d3",
"sector_size": 1024
},
"v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": {
"cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG",
"digest": "b2f09a0ccb62da28c890d5b881c8dcd2",
"sector_size": 1024
},
"v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": {
"cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q",
"digest": "b53c1916a63839ec345aa2224e9198b7",
"sector_size": 1073741824
},
"v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": {
"cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA",
"digest": "20d58a3fae7343481f8298a2dd493dd7",
"sector_size": 1073741824
},
"v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": {
"cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU",
"digest": "7698426202c7e07b26ef056d31485b3a",
"sector_size": 16777216
},
"v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": {
"cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax",
"digest": "834408e5c3fce6ec5d1bf64e64cee94e",
"sector_size": 16777216
}
}

95
build/proof-params/paramfetch.sh.template
View File

@ -1,95 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
PARAMS='{{PARAMSJSON}}'
die() {
echo "$@" >&2
exit 1
}
have_binary() {
type "$1" > /dev/null 2> /dev/null
}
check_writable() {
printf "" > "$1" && rm "$1"
}
try_download() {
url="$1"
output="$2"
command="$3"
util_name="$(set -- $command; echo "$1")"
if ! have_binary "$util_name"; then
return 1
fi
printf '==> Using %s to download "%s" to "%s"\n' "$util_name" "$url" "$output"
if eval "$command"; then
echo "==> Download complete!"
return
else
echo "error: couldn't download with $util_name ($?)"
return 1
fi
}
download() {
dl_url="$1"
dl_output="$2"
test "$#" -eq "2" || die "download requires exactly two arguments, was given $@"
if ! check_writable "$dl_output"; then
die "download error: cannot write to $dl_output"
fi
try_download "$dl_url" "$dl_output" "wget '$dl_url' -O '$dl_output'" && return
try_download "$dl_url" "$dl_output" "curl --silent --fail --output '$dl_output' '$dl_url'" && return
try_download "$dl_url" "$dl_output" "fetch '$dl_url' -o '$dl_output'" && return
try_download "$dl_url" "$dl_output" "http '$dl_url' > '$dl_output'" && return
try_download "$dl_url" "$dl_output" "ftp -o '$dl_output' '$dl_url'" && return
die "Unable to download $dl_url. exiting."
}
fetch_ipget() {
local dest="$1"
local cid="$2"
IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i"
./bin/ipget $IPGET_PARAMS -o "$dest" "$cid"
}
fetch_gateway() {
local dest="$1"
local cid="$2"
local url="http://198.211.99.118/ipfs/$cid"
download "$url" "$dest"
}
OUT_DIR="/var/tmp/filecoin-proof-parameters"
mkdir -p $OUT_DIR
printf $PARAMS | base64 -d | jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output | \
while read -r dest cid digest; do
if [[ -f "$dest" ]]; then
b2=$(b2sum "$dest" | head -c 32)
if [[ "$digest" == "$b2" ]]; then
echo "$dest exists and has correct hash"
continue
else
echo "$dest has incorrect hash"
rm -f "$dest"
fi
fi
echo "downloading $dest"
fetch_gateway "$dest" "$cid"
done

2
chain/actors/actor_miner.go
View File

@ -9,10 +9,10 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/go-amt-ipld"
"github.com/ipfs/go-cid"

2
chain/actors/actor_miner_test.go
View File

@ -7,12 +7,12 @@ import (
"testing"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
hamt "github.com/ipfs/go-hamt-ipld"
blockstore "github.com/ipfs/go-ipfs-blockstore"
cbg "github.com/whyrusleeping/cbor-gen"

2
chain/actors/actor_storagemarket.go
View File

@ -14,10 +14,10 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type StorageMarketActor struct{}

2
chain/deals/client.go
View File

@ -12,6 +12,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/events"
@ -20,7 +21,6 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/retrieval/discovery"

7
chain/deals/client_utils.go
View File

@ -5,6 +5,7 @@ import (
"context"
"runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
unixfile "github.com/ipfs/go-unixfs/file"
@ -12,11 +13,11 @@ import (
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-cbor-util"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)

2
chain/deals/provider.go
View File

@ -14,11 +14,11 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"

2
chain/deals/provider_utils.go
View File

@ -12,9 +12,9 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/ipfs/go-cid"
inet "github.com/libp2p/go-libp2p-core/network"

2
chain/deals/request_validation_test.go
View File

@ -15,11 +15,11 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/statestore"
)
var blockGenerator = blocksutil.NewBlockGenerator()

2
chain/gen/gen.go
View File

@ -11,6 +11,7 @@ import (
ffi "github.com/filecoin-project/filecoin-ffi"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-car"
offline "github.com/ipfs/go-ipfs-exchange-offline"
@ -28,7 +29,6 @@ import (
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/repo"
block "github.com/ipfs/go-block-format"

3
chain/stmgr/utils.go
View File

@ -4,12 +4,13 @@ import (
"context"
ffi "github.com/filecoin-project/filecoin-ffi"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
amt "github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"

3
chain/sync.go
View File

@ -12,6 +12,7 @@ import (
"github.com/Gurpartap/async"
bls "github.com/filecoin-project/filecoin-ffi"
amt "github.com/filecoin-project/go-amt-ipld"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
@ -26,6 +27,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
@ -35,7 +37,6 @@ import (
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("chain")

9
chain/types/blockheader.go
View File

@ -5,6 +5,8 @@ import (
"context"
"math/big"
"github.com/filecoin-project/go-sectorbuilder"
block "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/minio/sha256-simd"
@ -13,6 +15,7 @@ import (
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
)
@ -214,11 +217,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big
}
func ElectionPostChallengeCount(sectors uint64, faults int) uint64 {
if sectors == 0 {
return 0
}
// ceil(sectors / build.SectorChallengeRatioDiv)
return (sectors-uint64(faults)-1)/build.SectorChallengeRatioDiv + 1
return sectorbuilder.ElectionPostChallengeCount(sectors, faults)
}
func (t *Ticket) Equals(ot *Ticket) bool {

2
cmd/lotus-bench/main.go
View File

@ -23,10 +23,10 @@ import (
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("lotus-bench")

2
cmd/lotus-seal-worker/sub.go
View File

@ -7,8 +7,8 @@ import (
paramfetch "github.com/filecoin-project/go-paramfetch"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-sectorbuilder"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type worker struct {

2
cmd/lotus-seal-worker/transfer.go
View File

@ -6,12 +6,12 @@ import (
"net/http"
"os"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
files "github.com/ipfs/go-ipfs-files"
"golang.org/x/xerrors"
"gopkg.in/cheggaaa/pb.v1"
"path/filepath"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/tarutil"
)

2
cmd/lotus-seed/main.go
View File

@ -7,6 +7,7 @@ import (
"encoding/json"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"github.com/mitchellh/go-homedir"
@ -17,7 +18,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("lotus-seed")

2
cmd/lotus-seed/seed/seed.go
View File

@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
@ -20,7 +21,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("preseal")

2
cmd/lotus-storage-miner/init.go
View File

@ -22,6 +22,7 @@ import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-sectorbuilder"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
@ -29,7 +30,6 @@ import (
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"

15
go.mod
View File

@ -9,18 +9,18 @@ require (
github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/docker/go-units v0.4.0
github.com/fatih/color v1.7.0 // indirect
github.com/filecoin-project/chain-validation v0.0.3
github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed
github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590
github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/google/go-cmp v0.3.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.1
github.com/hashicorp/go-multierror v1.0.0
@ -69,12 +69,10 @@ require (
github.com/libp2p/go-libp2p-tls v0.1.0
github.com/libp2p/go-libp2p-yamux v0.2.1
github.com/libp2p/go-maddr-filter v0.0.5
github.com/mattn/go-isatty v0.0.9 // indirect
github.com/mattn/go-sqlite3 v1.12.0
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/minio/sha256-simd v0.1.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mr-tron/base58 v1.1.3 // indirect
github.com/multiformats/go-base32 v0.0.3
github.com/multiformats/go-multiaddr v0.1.1
github.com/multiformats/go-multiaddr-dns v0.2.0
@ -84,24 +82,19 @@ require (
github.com/onsi/ginkgo v1.9.0 // indirect
github.com/onsi/gomega v1.6.0 // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/otiai10/copy v1.0.2
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
github.com/smartystreets/assertions v1.0.1 // indirect
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
go.opencensus.io v0.22.1
go.opencensus.io v0.22.2
go.uber.org/dig v1.7.0 // indirect
go.uber.org/fx v1.9.0
go.uber.org/goleak v0.10.0 // indirect
go.uber.org/multierr v1.4.0
go.uber.org/zap v1.10.0
go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
gopkg.in/cheggaaa/pb.v1 v1.0.28

16
go.sum
View File

@ -85,12 +85,16 @@ github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2j
github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cODZD2YzpTUtrOSxbEnWFcQHidNRZiRdvLxySjGvG/M=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY=
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
@ -179,6 +183,7 @@ github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds=
github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@ -530,6 +535,7 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
@ -660,6 +666,8 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=

126
lib/sectorbuilder/files.go
View File

@ -1,126 +0,0 @@
package sectorbuilder
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"golang.org/x/xerrors"
)
func (sb *SectorBuilder) SectorName(sectorID uint64) string {
return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID)
}
func (sb *SectorBuilder) StagedSectorPath(sectorID uint64) string {
return filepath.Join(sb.filesystem.pathFor(dataStaging), sb.SectorName(sectorID))
}
func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string {
return filepath.Join(sb.filesystem.pathFor(dataUnsealed), sb.SectorName(sectorID))
}
func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
return os.OpenFile(sb.StagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
}
func (sb *SectorBuilder) SealedSectorPath(sectorID uint64) (string, error) {
path := filepath.Join(sb.filesystem.pathFor(dataSealed), sb.SectorName(sectorID))
return path, nil
}
func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) {
dir := filepath.Join(sb.filesystem.pathFor(dataCache), sb.SectorName(sectorID))
err := os.Mkdir(dir, 0755)
if os.IsExist(err) {
err = nil
}
return dir, err
}
func (sb *SectorBuilder) GetPath(typ string, sectorName string) (string, error) {
_, found := overheadMul[dataType(typ)]
if !found {
return "", xerrors.Errorf("unknown sector type: %s", typ)
}
return filepath.Join(sb.filesystem.pathFor(dataType(typ)), sectorName), nil
}
func (sb *SectorBuilder) TrimCache(sectorID uint64) error {
dir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return xerrors.Errorf("getting cache dir: %w", err)
}
files, err := ioutil.ReadDir(dir)
if err != nil {
return xerrors.Errorf("readdir: %w", err)
}
for _, file := range files {
if !strings.HasSuffix(file.Name(), ".dat") { // _aux probably
continue
}
if strings.HasSuffix(file.Name(), "-data-tree-r-last.dat") { // Want to keep
continue
}
if err := os.Remove(filepath.Join(dir, file.Name())); err != nil {
return xerrors.Errorf("rm %s: %w", file.Name(), err)
}
}
return nil
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var wait sync.Mutex
var werr error
wait.Lock()
go func() {
defer wait.Unlock()
var copied int64
copied, werr = io.CopyN(w, r, n)
if werr != nil {
log.Warnf("toReadableFile: copy error: %+v", werr)
}
err := w.Close()
if werr == nil && err != nil {
werr = err
log.Warnf("toReadableFile: close error: %+v", err)
return
}
if copied != n {
log.Warnf("copied different amount than expected: %d != %d", copied, n)
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
}
}()
return f, func() error {
wait.Lock()
return werr
}, nil
}

116
lib/sectorbuilder/fs.go
View File

@ -1,116 +0,0 @@
package sectorbuilder
import (
"github.com/filecoin-project/lotus/chain/types"
"golang.org/x/xerrors"
"os"
"path/filepath"
"sync"
"syscall"
)
type dataType string
const (
dataCache dataType = "cache"
dataStaging dataType = "staging"
dataSealed dataType = "sealed"
dataUnsealed dataType = "unsealed"
)
var overheadMul = map[dataType]uint64{ // * sectorSize
dataCache: 11, // TODO: check if true for 32G sectors
dataStaging: 1,
dataSealed: 1,
dataUnsealed: 1,
}
type fs struct {
path string
// in progress actions
reserved map[dataType]uint64
lk sync.Mutex
}
func openFs(dir string) *fs {
return &fs{
path: dir,
reserved: map[dataType]uint64{},
}
}
func (f *fs) init() error {
for _, dir := range []string{f.path,
f.pathFor(dataCache),
f.pathFor(dataStaging),
f.pathFor(dataSealed),
f.pathFor(dataUnsealed)} {
if err := os.Mkdir(dir, 0755); err != nil {
if os.IsExist(err) {
continue
}
return err
}
}
return nil
}
func (f *fs) pathFor(typ dataType) string {
_, found := overheadMul[typ]
if !found {
panic("unknown data path requested")
}
return filepath.Join(f.path, string(typ))
}
func (f *fs) reservedBytes() int64 {
var out int64
for _, r := range f.reserved {
out += int64(r)
}
return out
}
func (f *fs) reserve(typ dataType, size uint64) error {
f.lk.Lock()
defer f.lk.Unlock()
var fsstat syscall.Statfs_t
if err := syscall.Statfs(f.pathFor(typ), &fsstat); err != nil {
return err
}
fsavail := int64(fsstat.Bavail) * int64(fsstat.Bsize)
avail := fsavail - f.reservedBytes()
need := overheadMul[typ] * size
if int64(need) > avail {
return xerrors.Errorf("not enough space in '%s', need %s, available %s (fs: %s, reserved: %s)",
f.path,
types.NewInt(need).SizeStr(),
types.NewInt(uint64(avail)).SizeStr(),
types.NewInt(uint64(fsavail)).SizeStr(),
types.NewInt(uint64(f.reservedBytes())).SizeStr())
}
f.reserved[typ] += need
return nil
}
func (f *fs) free(typ dataType, sectorSize uint64) {
f.lk.Lock()
defer f.lk.Unlock()
f.reserved[typ] -= overheadMul[typ] * sectorSize
return
}

27
lib/sectorbuilder/mock.go
View File

@ -1,27 +0,0 @@
package sectorbuilder
import (
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) {
addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq")
if err != nil {
return nil, err
}
sb, err := New(&Config{
SectorSize: sectorSize,
Dir: dir,
WorkerThreads: 2,
Miner: addr,
}, ds)
if err != nil {
return nil, err
}
return sb, nil
}

174
lib/sectorbuilder/remote.go
View File

@ -1,174 +0,0 @@
package sectorbuilder
import (
"context"
"golang.org/x/xerrors"
)
type WorkerTaskType int
const (
WorkerPreCommit WorkerTaskType = iota
WorkerCommit
)
type WorkerTask struct {
Type WorkerTaskType
TaskID uint64
SectorID uint64
// preCommit
SealTicket SealTicket
Pieces []PublicPieceInfo
// commit
SealSeed SealSeed
Rspco RawSealPreCommitOutput
}
type workerCall struct {
task WorkerTask
ret chan SealRes
}
func (sb *SectorBuilder) AddWorker(ctx context.Context, cfg WorkerCfg) (<-chan WorkerTask, error) {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
taskCh := make(chan WorkerTask)
r := &remote{
sealTasks: taskCh,
busy: 0,
}
sb.remoteCtr++
sb.remotes[sb.remoteCtr] = r
go sb.remoteWorker(ctx, r, cfg)
return taskCh, nil
}
func (sb *SectorBuilder) returnTask(task workerCall) {
var ret chan workerCall
switch task.task.Type {
case WorkerPreCommit:
ret = sb.precommitTasks
case WorkerCommit:
ret = sb.commitTasks
default:
log.Error("unknown task type", task.task.Type)
}
go func() {
select {
case ret <- task:
case <-sb.stopping:
return
}
}()
}
func (sb *SectorBuilder) remoteWorker(ctx context.Context, r *remote, cfg WorkerCfg) {
defer log.Warn("Remote worker disconnected")
defer func() {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
for i, vr := range sb.remotes {
if vr == r {
delete(sb.remotes, i)
return
}
}
}()
precommits := sb.precommitTasks
if cfg.NoPreCommit {
precommits = nil
}
commits := sb.commitTasks
if cfg.NoCommit {
commits = nil
}
for {
select {
case task := <-commits:
sb.doTask(ctx, r, task)
case task := <-precommits:
sb.doTask(ctx, r, task)
case <-ctx.Done():
return
case <-sb.stopping:
return
}
r.lk.Lock()
r.busy = 0
r.lk.Unlock()
}
}
func (sb *SectorBuilder) doTask(ctx context.Context, r *remote, task workerCall) {
resCh := make(chan SealRes)
sb.remoteLk.Lock()
sb.remoteResults[task.task.TaskID] = resCh
sb.remoteLk.Unlock()
// send the task
select {
case r.sealTasks <- task.task:
case <-ctx.Done():
sb.returnTask(task)
return
}
r.lk.Lock()
r.busy = task.task.TaskID
r.lk.Unlock()
// wait for the result
select {
case res := <-resCh:
// send the result back to the caller
select {
case task.ret <- res:
case <-ctx.Done():
return
case <-sb.stopping:
return
}
case <-ctx.Done():
log.Warnf("context expired while waiting for task %d (sector %d): %s", task.task.TaskID, task.task.SectorID, ctx.Err())
return
case <-sb.stopping:
return
}
}
func (sb *SectorBuilder) TaskDone(ctx context.Context, task uint64, res SealRes) error {
sb.remoteLk.Lock()
rres, ok := sb.remoteResults[task]
if ok {
delete(sb.remoteResults, task)
}
sb.remoteLk.Unlock()
if !ok {
return xerrors.Errorf("task %d not found", task)
}
select {
case rres <- res:
return nil
case <-ctx.Done():
return ctx.Err()
}
}

84
lib/sectorbuilder/scrub.go
View File

@ -1,84 +0,0 @@
package sectorbuilder
import (
"io/ioutil"
"os"
"path/filepath"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"golang.org/x/xerrors"
)
type Fault struct {
SectorID uint64
Err error
}
func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault {
var faults []*Fault
for _, sector := range sectorSet.Values() {
err := sb.checkSector(sector.SectorID)
if err != nil {
faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err})
}
}
return faults
}
func (sb *SectorBuilder) checkSector(sectorID uint64) error {
cache, err := sb.sectorCacheDir(sectorID)
if err != nil {
return xerrors.Errorf("getting sector cache dir: %w", err)
}
if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil {
return err
}
if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil {
return err
}
// TODO: better validate this
if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be?
return err
}
dent, err := ioutil.ReadDir(cache)
if err != nil {
return xerrors.Errorf("reading cache dir %s", cache)
}
if len(dent) != 3 {
return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache)
}
sealed, err := sb.SealedSectorPath(sectorID)
if err != nil {
return xerrors.Errorf("getting sealed sector path: %w", err)
}
if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil {
return err
}
return nil
}
func assertFile(path string, minSz uint64, maxSz uint64) error {
st, err := os.Stat(path)
if err != nil {
return xerrors.Errorf("stat %s: %w", path, err)
}
if st.IsDir() {
return xerrors.Errorf("expected %s to be a regular file", path)
}
if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz {
return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size())
}
return nil
}

799
lib/sectorbuilder/sectorbuilder.go
View File

@ -1,799 +0,0 @@
package sectorbuilder
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
dcopy "github.com/otiai10/copy"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
const PoStReservedWorkers = 1
const PoRepProofPartitions = 10
var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
var log = logging.Logger("sectorbuilder")
type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo
type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo
type SealTicket = sectorbuilder.SealTicket
type SealSeed = sectorbuilder.SealSeed
type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput
type SealCommitOutput = sectorbuilder.SealCommitOutput
type PublicPieceInfo = sectorbuilder.PublicPieceInfo
type RawSealPreCommitOutput sectorbuilder.RawSealPreCommitOutput
type EPostCandidate = sectorbuilder.Candidate
const CommLen = sectorbuilder.CommitmentBytesLen
type WorkerCfg struct {
NoPreCommit bool
NoCommit bool
// TODO: 'cost' info, probably in terms of sealing + transfer speed
}
type SectorBuilder struct {
ds dtypes.MetadataDS
idLk sync.Mutex
ssize uint64
lastID uint64
Miner address.Address
unsealLk sync.Mutex
noCommit bool
noPreCommit bool
rateLimit chan struct{}
precommitTasks chan workerCall
commitTasks chan workerCall
taskCtr uint64
remoteLk sync.Mutex
remoteCtr int
remotes map[int]*remote
remoteResults map[uint64]chan<- SealRes
addPieceWait int32
preCommitWait int32
commitWait int32
unsealWait int32
fsLk sync.Mutex
filesystem *fs // TODO: multi-fs support
stopping chan struct{}
}
type JsonRSPCO struct {
CommD []byte
CommR []byte
}
func (rspco *RawSealPreCommitOutput) ToJson() JsonRSPCO {
return JsonRSPCO{
CommD: rspco.CommD[:],
CommR: rspco.CommR[:],
}
}
func (rspco *JsonRSPCO) rspco() RawSealPreCommitOutput {
var out RawSealPreCommitOutput
copy(out.CommD[:], rspco.CommD)
copy(out.CommR[:], rspco.CommR)
return out
}
type SealRes struct {
Err string
GoErr error `json:"-"`
Proof []byte
Rspco JsonRSPCO
}
type remote struct {
lk sync.Mutex
sealTasks chan<- WorkerTask
busy uint64 // only for metrics
}
type Config struct {
SectorSize uint64
Miner address.Address
WorkerThreads uint8
FallbackLastID uint64
NoCommit bool
NoPreCommit bool
Dir string
_ struct{} // guard against nameless init
}
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
if cfg.WorkerThreads < PoStReservedWorkers {
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers, cfg.WorkerThreads)
}
var lastUsedID uint64
b, err := ds.Get(lastSectorIdKey)
switch err {
case nil:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return nil, err
}
lastUsedID = uint64(i)
case datastore.ErrNotFound:
lastUsedID = cfg.FallbackLastID
default:
return nil, err
}
rlimit := cfg.WorkerThreads - PoStReservedWorkers
sealLocal := rlimit > 0
if rlimit == 0 {
rlimit = 1
}
sb := &SectorBuilder{
ds: ds,
ssize: cfg.SectorSize,
lastID: lastUsedID,
filesystem: openFs(cfg.Dir),
Miner: cfg.Miner,
noPreCommit: cfg.NoPreCommit || !sealLocal,
noCommit: cfg.NoCommit || !sealLocal,
rateLimit: make(chan struct{}, rlimit),
taskCtr: 1,
precommitTasks: make(chan workerCall),
commitTasks: make(chan workerCall),
remoteResults: map[uint64]chan<- SealRes{},
remotes: map[int]*remote{},
stopping: make(chan struct{}),
}
if err := sb.filesystem.init(); err != nil {
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
}
return sb, nil
}
func NewStandalone(cfg *Config) (*SectorBuilder, error) {
sb := &SectorBuilder{
ds: nil,
ssize: cfg.SectorSize,
Miner: cfg.Miner,
filesystem: openFs(cfg.Dir),
taskCtr: 1,
remotes: map[int]*remote{},
rateLimit: make(chan struct{}, cfg.WorkerThreads),
stopping: make(chan struct{}),
}
if err := sb.filesystem.init(); err != nil {
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
}
return sb, nil
}
func (sb *SectorBuilder) checkRateLimit() {
if cap(sb.rateLimit) == len(sb.rateLimit) {
log.Warn("rate-limiting local sectorbuilder call")
}
}
func (sb *SectorBuilder) RateLimit() func() {
sb.checkRateLimit()
sb.rateLimit <- struct{}{}
return func() {
<-sb.rateLimit
}
}
type WorkerStats struct {
LocalFree int
LocalReserved int
LocalTotal int
// todo: post in progress
RemotesTotal int
RemotesFree int
AddPieceWait int
PreCommitWait int
CommitWait int
UnsealWait int
}
func (sb *SectorBuilder) WorkerStats() WorkerStats {
sb.remoteLk.Lock()
defer sb.remoteLk.Unlock()
remoteFree := len(sb.remotes)
for _, r := range sb.remotes {
if r.busy > 0 {
remoteFree--
}
}
return WorkerStats{
LocalFree: cap(sb.rateLimit) - len(sb.rateLimit),
LocalReserved: PoStReservedWorkers,
LocalTotal: cap(sb.rateLimit) + PoStReservedWorkers,
RemotesTotal: len(sb.remotes),
RemotesFree: remoteFree,
AddPieceWait: int(atomic.LoadInt32(&sb.addPieceWait)),
PreCommitWait: int(atomic.LoadInt32(&sb.preCommitWait)),
CommitWait: int(atomic.LoadInt32(&sb.commitWait)),
UnsealWait: int(atomic.LoadInt32(&sb.unsealWait)),
}
}
func addressToProverID(a address.Address) [32]byte {
var proverId [32]byte
copy(proverId[:], a.Payload())
return proverId
}
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
sb.idLk.Lock()
defer sb.idLk.Unlock()
sb.lastID++
id := sb.lastID
err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
if err != nil {
return 0, err
}
return id, nil
}
func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) {
fs := sb.filesystem
if err := fs.reserve(dataStaging, sb.ssize); err != nil {
return PublicPieceInfo{}, err
}
defer fs.free(dataStaging, sb.ssize)
atomic.AddInt32(&sb.addPieceWait, 1)
ret := sb.RateLimit()
atomic.AddInt32(&sb.addPieceWait, -1)
defer ret()
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return PublicPieceInfo{}, err
}
stagedFile, err := sb.stagedSectorFile(sectorId)
if err != nil {
return PublicPieceInfo{}, err
}
_, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
if err != nil {
return PublicPieceInfo{}, err
}
if err := stagedFile.Close(); err != nil {
return PublicPieceInfo{}, err
}
if err := f.Close(); err != nil {
return PublicPieceInfo{}, err
}
return PublicPieceInfo{
Size: pieceSize,
CommP: commP,
}, werr()
}
func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
fs := sb.filesystem
if err := fs.reserve(dataUnsealed, sb.ssize); err != nil { // TODO: this needs to get smarter when we start supporting partial unseals
return nil, err
}
defer fs.free(dataUnsealed, sb.ssize)
atomic.AddInt32(&sb.unsealWait, 1)
// TODO: Don't wait if cached
ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker
defer ret()
atomic.AddInt32(&sb.unsealWait, -1)
sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel
defer sb.unsealLk.Unlock()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
sealedPath, err := sb.SealedSectorPath(sectorID)
if err != nil {
return nil, err
}
unsealedPath := sb.unsealedSectorPath(sectorID)
// TODO: GC for those
// (Probably configurable count of sectors to be kept unsealed, and just
// remove last used one (or use whatever other cache policy makes sense))
f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
var commd [CommLen]byte
copy(commd[:], commD)
var tkt [CommLen]byte
copy(tkt[:], ticket)
err = sectorbuilder.Unseal(sb.ssize,
PoRepProofPartitions,
cacheDir,
sealedPath,
unsealedPath,
sectorID,
addressToProverID(sb.Miner),
tkt,
commd)
if err != nil {
return nil, xerrors.Errorf("unseal failed: %w", err)
}
f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
return nil, err
}
}
if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
return nil, xerrors.Errorf("seek: %w", err)
}
lr := io.LimitReader(f, int64(size))
return &struct {
io.Reader
io.Closer
}{
Reader: lr,
Closer: f,
}, nil
}
func (sb *SectorBuilder) sealPreCommitRemote(call workerCall) (RawSealPreCommitOutput, error) {
atomic.AddInt32(&sb.preCommitWait, -1)
select {
case ret := <-call.ret:
var err error
if ret.Err != "" {
err = xerrors.New(ret.Err)
}
return ret.Rspco.rspco(), err
case <-sb.stopping:
return RawSealPreCommitOutput{}, xerrors.New("sectorbuilder stopped")
}
}
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
fs := sb.filesystem
if err := fs.reserve(dataCache, sb.ssize); err != nil {
return RawSealPreCommitOutput{}, err
}
defer fs.free(dataCache, sb.ssize)
if err := fs.reserve(dataSealed, sb.ssize); err != nil {
return RawSealPreCommitOutput{}, err
}
defer fs.free(dataSealed, sb.ssize)
call := workerCall{
task: WorkerTask{
Type: WorkerPreCommit,
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
SectorID: sectorID,
SealTicket: ticket,
Pieces: pieces,
},
ret: make(chan SealRes),
}
atomic.AddInt32(&sb.preCommitWait, 1)
select { // prefer remote
case sb.precommitTasks <- call:
return sb.sealPreCommitRemote(call)
default:
}
sb.checkRateLimit()
rl := sb.rateLimit
if sb.noPreCommit {
rl = make(chan struct{})
}
select { // use whichever is available
case sb.precommitTasks <- call:
return sb.sealPreCommitRemote(call)
case rl <- struct{}{}:
}
atomic.AddInt32(&sb.preCommitWait, -1)
// local
defer func() {
<-sb.rateLimit
}()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("getting cache dir: %w", err)
}
sealedPath, err := sb.SealedSectorPath(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("getting sealed sector path: %w", err)
}
e, err := os.OpenFile(sealedPath, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("ensuring sealed file exists: %w", err)
}
if err := e.Close(); err != nil {
return RawSealPreCommitOutput{}, err
}
var sum uint64
for _, piece := range pieces {
sum += piece.Size
}
ussize := UserBytesForSectorSize(sb.ssize)
if sum != ussize {
return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
}
stagedPath := sb.StagedSectorPath(sectorID)
rspco, err := sectorbuilder.SealPreCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
stagedPath,
sealedPath,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
pieces,
)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err)
}
return RawSealPreCommitOutput(rspco), nil
}
func (sb *SectorBuilder) sealCommitRemote(call workerCall) (proof []byte, err error) {
atomic.AddInt32(&sb.commitWait, -1)
select {
case ret := <-call.ret:
if ret.Err != "" {
err = xerrors.New(ret.Err)
}
return ret.Proof, err
case <-sb.stopping:
return nil, xerrors.New("sectorbuilder stopped")
}
}
func (sb *SectorBuilder) sealCommitLocal(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
atomic.AddInt32(&sb.commitWait, -1)
defer func() {
<-sb.rateLimit
}()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
proof, err = sectorbuilder.SealCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
seed.TicketBytes,
pieces,
sectorbuilder.RawSealPreCommitOutput(rspco),
)
if err != nil {
log.Warn("StandaloneSealCommit error: ", err)
log.Warnf("sid:%d tkt:%v seed:%v, ppi:%v rspco:%v", sectorID, ticket, seed, pieces, rspco)
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
call := workerCall{
task: WorkerTask{
Type: WorkerCommit,
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
SectorID: sectorID,
SealTicket: ticket,
Pieces: pieces,
SealSeed: seed,
Rspco: rspco,
},
ret: make(chan SealRes),
}
atomic.AddInt32(&sb.commitWait, 1)
select { // prefer remote
case sb.commitTasks <- call:
proof, err = sb.sealCommitRemote(call)
default:
sb.checkRateLimit()
rl := sb.rateLimit
if sb.noCommit {
rl = make(chan struct{})
}
select { // use whichever is available
case sb.commitTasks <- call:
proof, err = sb.sealCommitRemote(call)
case rl <- struct{}{}:
proof, err = sb.sealCommitLocal(sectorID, ticket, seed, pieces, rspco)
}
}
if err != nil {
return nil, xerrors.Errorf("commit: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) {
if len(challengeSeed) != CommLen {
return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen)
}
var cseed [CommLen]byte
copy(cseed[:], challengeSeed)
privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
}
func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
if err != nil {
return nil, err
}
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
}
func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) {
fmap := map[uint64]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []sectorbuilder.PrivateSectorInfo
for _, s := range sectorInfo.Values() {
if _, faulty := fmap[s.SectorID]; faulty {
continue
}
cachePath, err := sb.sectorCacheDir(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err)
}
sealedPath, err := sb.SealedSectorPath(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err)
}
out = append(out, sectorbuilder.PrivateSectorInfo{
SectorID: s.SectorID,
CommR: s.CommR,
CacheDirPath: cachePath,
SealedSectorPath: sealedPath,
})
}
return NewSortedPrivateSectorInfo(out), nil
}
func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
if err != nil {
return nil, nil, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
proverID := addressToProverID(sb.Miner)
candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, nil, err
}
proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates)
return candidates, proof, err
}
func (sb *SectorBuilder) Stop() {
close(sb.stopping)
}
func fallbackPostChallengeCount(sectors uint64, faults int) uint64 {
challengeCount := types.ElectionPostChallengeCount(sectors, faults)
if challengeCount > build.MaxFallbackPostChallengeCount {
return build.MaxFallbackPostChallengeCount
}
return challengeCount
}
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
if err := migrate(osb.filesystem.pathFor(dataCache), sb.filesystem.pathFor(dataCache), symlink); err != nil {
return err
}
if err := migrate(osb.filesystem.pathFor(dataStaging), sb.filesystem.pathFor(dataStaging), symlink); err != nil {
return err
}
if err := migrate(osb.filesystem.pathFor(dataSealed), sb.filesystem.pathFor(dataSealed), symlink); err != nil {
return err
}
val, err := osb.ds.Get(lastSectorIdKey)
if err != nil {
return err
}
if err := sb.ds.Put(lastSectorIdKey, val); err != nil {
return err
}
sb.lastID = osb.lastID
return nil
}
func (sb *SectorBuilder) SetLastSectorID(id uint64) error {
if err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))); err != nil {
return err
}
sb.lastID = id
return nil
}
func migrate(from, to string, symlink bool) error {
st, err := os.Stat(from)
if err != nil {
return err
}
if st.IsDir() {
return migrateDir(from, to, symlink)
}
return migrateFile(from, to, symlink)
}
func migrateDir(from, to string, symlink bool) error {
tost, err := os.Stat(to)
if err != nil {
if !os.IsNotExist(err) {
return err
}
if err := os.MkdirAll(to, 0755); err != nil {
return err
}
} else if !tost.IsDir() {
return xerrors.Errorf("target %q already exists and is a file (expected directory)")
}
dirents, err := ioutil.ReadDir(from)
if err != nil {
return err
}
for _, inf := range dirents {
n := inf.Name()
if inf.IsDir() {
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
return err
}
} else {
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
return err
}
}
}
return nil
}
func migrateFile(from, to string, symlink bool) error {
if symlink {
return os.Symlink(from, to)
}
return dcopy.Copy(from, to)
}

365
lib/sectorbuilder/sectorbuilder_test.go
View File

@ -1,365 +0,0 @@
package sectorbuilder_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"runtime"
"sync"
"testing"
"time"
ffi "github.com/filecoin-project/filecoin-ffi"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func init() {
logging.SetLogLevel("*", "INFO")
}
const sectorSize = 1024
type seal struct {
sid uint64
pco sectorbuilder.RawSealPreCommitOutput
ppi sectorbuilder.PublicPieceInfo
ticket sectorbuilder.SealTicket
}
func (s *seal) precommit(t *testing.T, sb *sectorbuilder.SectorBuilder, sid uint64, done func()) {
dlen := sectorbuilder.UserBytesForSectorSize(sectorSize)
var err error
r := io.LimitReader(rand.New(rand.NewSource(42+int64(sid))), int64(dlen))
s.ppi, err = sb.AddPiece(dlen, sid, r, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
s.ticket = sectorbuilder.SealTicket{
BlockHeight: 5,
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
}
s.pco, err = sb.SealPreCommit(sid, s.ticket, []sectorbuilder.PublicPieceInfo{s.ppi})
if err != nil {
t.Fatalf("%+v", err)
}
done()
}
func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()) {
seed := sectorbuilder.SealSeed{
BlockHeight: 15,
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
}
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifySeal(sectorSize, s.pco.CommR[:], s.pco.CommD[:], sb.Miner, s.ticket.TicketBytes[:], seed.TicketBytes[:], s.sid, proof)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("proof failed to validate")
}
done()
}
func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time {
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
ppi := make([]ffi.PublicSectorInfo, len(seals))
for i, s := range seals {
ppi[i] = ffi.PublicSectorInfo{
SectorID: s.sid,
CommR: s.pco.CommR,
}
}
ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi)
candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
genCandidates := time.Now()
if len(candndates) != 1 {
t.Fatal("expected 1 candidate")
}
postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("bad post")
}
return genCandidates
}
func TestSealAndVerify(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
si, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
s.commit(t, sb, func() {})
commit := time.Now()
genCandidiates := post(t, sb, s)
epost := time.Now()
// Restart sectorbuilder, re-run post
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
post(t, sb, s)
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("Commit: %s\n", commit.Sub(precommit).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealPoStNoCommit(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
si, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
// Restart sectorbuilder, re-run post
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
if err := sb.TrimCache(1); err != nil {
t.Fatal(err)
}
genCandidiates := post(t, sb, s)
epost := time.Now()
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealAndVerify2(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode")
}
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := paramfetch.GetParams(sectorSize); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
var wg sync.WaitGroup
si1, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
si2, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s1 := seal{sid: si1}
s2 := seal{sid: si2}
wg.Add(2)
go s1.precommit(t, sb, 1, wg.Done)
time.Sleep(100 * time.Millisecond)
go s2.precommit(t, sb, 2, wg.Done)
wg.Wait()
wg.Add(2)
go s1.commit(t, sb, wg.Done)
go s2.commit(t, sb, wg.Done)
wg.Wait()
post(t, sb, s1, s2)
}
func TestAcquireID(t *testing.T) {
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire := func(expect uint64) {
id, err := sb.AcquireSectorId()
require.NoError(t, err)
assert.Equal(t, expect, id)
}
assertAcquire(1)
assertAcquire(2)
assertAcquire(3)
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire(4)
assertAcquire(5)
assertAcquire(6)
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}

75
lib/sectorbuilder/simple.go
View File

@ -1,75 +0,0 @@
package sectorbuilder
import (
"context"
"io"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"go.opencensus.io/trace"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/chain/types"
)
func (sb *SectorBuilder) SectorSize() uint64 {
return sb.ssize
}
var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
var commRa, commDa, ticketa, seeda [32]byte
copy(commRa[:], commR)
copy(commDa[:], commD)
copy(ticketa[:], ticket)
copy(seeda[:], seed)
proverIDa := addressToProverID(proverID)
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
}
func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo {
return sectorbuilder.NewSortedPrivateSectorInfo(sectors...)
}
func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo {
return sectorbuilder.NewSortedPublicSectorInfo(sectors...)
}
func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0)
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) {
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults)
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
var challengeSeeda [CommLen]byte
copy(challengeSeeda[:], challengeSeed)
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
prover := addressToProverID(proverID)
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover)
}
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
f, werr, err := toReadableFile(piece, int64(pieceSize))
if err != nil {
return [32]byte{}, err
}
commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
if err != nil {
return [32]byte{}, err
}
return commP, werr()
}
func GenerateDataCommitment(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([CommLen]byte, error) {
return sectorbuilder.GenerateDataCommitment(ssize, pieces)
}

165
lib/statestore/store.go
View File

@ -1,165 +0,0 @@
package statestore
import (
"bytes"
"fmt"
"reflect"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
cbg "github.com/whyrusleeping/cbor-gen"
"go.uber.org/multierr"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-cbor-util"
)
type StateStore struct {
ds datastore.Datastore
}
func New(ds datastore.Datastore) *StateStore {
return &StateStore{ds: ds}
}
func toKey(k interface{}) datastore.Key {
switch t := k.(type) {
case uint64:
return datastore.NewKey(fmt.Sprint(t))
case fmt.Stringer:
return datastore.NewKey(t.String())
default:
panic("unexpected key type")
}
}
func (st *StateStore) Begin(i interface{}, state interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if has {
return xerrors.Errorf("already tracking state for %v", i)
}
b, err := cborutil.Dump(state)
if err != nil {
return err
}
return st.ds.Put(k, b)
}
func (st *StateStore) End(i interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
return st.ds.Delete(k)
}
func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
rmut := reflect.ValueOf(mutator)
return func(in []byte) ([]byte, error) {
state := reflect.New(rmut.Type().In(0).Elem())
err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
if err != nil {
return nil, err
}
out := rmut.Call([]reflect.Value{state})
if err := out[0].Interface(); err != nil {
return nil, err.(error)
}
return cborutil.Dump(state.Interface())
}
}
// mutator func(*T) error
func (st *StateStore) Mutate(i interface{}, mutator interface{}) error {
return st.mutate(i, cborMutator(mutator))
}
func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
cur, err := st.ds.Get(k)
if err != nil {
return err
}
mutated, err := mutator(cur)
if err != nil {
return err
}
return st.ds.Put(k, mutated)
}
func (st *StateStore) Has(i interface{}) (bool, error) {
return st.ds.Has(toKey(i))
}
func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
k := toKey(i)
val, err := st.ds.Get(k)
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
return xerrors.Errorf("No state for %s: %w", i, err)
}
return err
}
return out.UnmarshalCBOR(bytes.NewReader(val))
}
// out: *[]T
func (st *StateStore) List(out interface{}) error {
res, err := st.ds.Query(query.Query{})
if err != nil {
return err
}
defer res.Close()
outT := reflect.TypeOf(out).Elem().Elem()
rout := reflect.ValueOf(out)
var errs error
for {
res, ok := res.NextSync()
if !ok {
break
}
if res.Error != nil {
return res.Error
}
elem := reflect.New(outT)
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
if err != nil {
errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
continue
}
rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
}
return nil
}

38
lib/statestore/store_test.go
View File

@ -1,38 +0,0 @@
package statestore
import (
"testing"
"github.com/ipfs/go-datastore"
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/chain/types"
)
func TestList(t *testing.T) {
ds := datastore.NewMapDatastore()
e, err := cborutil.Dump(types.NewInt(7))
if err != nil {
t.Fatal(err)
}
if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
t.Fatal(err)
}
st := &StateStore{ds: ds}
var out []types.BigInt
if err := st.List(&out); err != nil {
t.Fatal(err)
}
if len(out) != 1 {
t.Fatal("wrong len")
}
if out[0].Int64() != 7 {
t.Fatal("wrong data")
}
}

2
node/builder.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"time"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
blockstore "github.com/ipfs/go-ipfs-blockstore"
ci "github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/host"
@ -30,7 +31,6 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello"

2
node/impl/storminer.go
View File

@ -13,8 +13,8 @@ import (
files "github.com/ipfs/go-ipfs-files"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/tarutil"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/storage"

2
node/modules/client.go
View File

@ -5,7 +5,7 @@ import (
"path/filepath"
"reflect"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"

2
node/modules/dtypes/storage.go
View File

@ -8,8 +8,8 @@ import (
exchange "github.com/ipfs/go-ipfs-exchange-interface"
ipld "github.com/ipfs/go-ipld-format"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/statestore"
)
// MetadataDS stores metadata

4
node/modules/storageminer.go
View File

@ -20,13 +20,13 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"

3
node/node_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/libp2p/go-libp2p-core/crypto"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
@ -21,6 +22,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/lotus/api/test"
@ -30,7 +32,6 @@ import (
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/jsonrpc"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/impl"

2
storage/fpost_run.go
View File

@ -5,13 +5,13 @@ import (
"time"
ffi "github.com/filecoin-project/filecoin-ffi"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func (s *fpostScheduler) failPost(eps uint64) {

2
storage/fpost_sched.go
View File

@ -4,6 +4,7 @@ import (
"context"
"sync"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
@ -11,7 +12,6 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
const Inactive = 0

2
storage/garbage.go
View File

@ -7,12 +7,12 @@ import (
"math"
"math/rand"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {

4
storage/miner.go
View File

@ -13,14 +13,14 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-statestore"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
)
var log = logging.Logger("storageminer")

2
storage/sector_states.go
View File

@ -3,13 +3,13 @@ package storage
import (
"context"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type providerHandlerFunc func(ctx context.Context, deal SectorInfo) *sectorUpdate

2
storage/sector_types.go
View File

@ -3,10 +3,10 @@ package storage
import (
"context"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error)

2
storage/sector_utils.go
View File

@ -3,7 +3,7 @@ package storage
import (
"math/bits"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
)
func fillersFromRem(toFill uint64) ([]uint64, error) {

2
storage/sector_utils_test.go
View File

@ -5,7 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
)
func testFill(t *testing.T, n uint64, exp []uint64) {

2
storage/sectorblocks/blocks.go
View File

@ -7,6 +7,7 @@ import (
"io"
"sync"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
@ -21,7 +22,6 @@ import (
"github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage"
)

2
storage/sectors.go
View File

@ -6,11 +6,11 @@ import (
"io"
"math"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
const NonceIncrement = math.MaxUint64