diff --git a/.circleci/config.yml b/.circleci/config.yml index 43e609d00..b9aa747df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,14 +37,13 @@ commands: - restore_cache: name: Restore parameters cache keys: - - 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' - - 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-' + - 'v20-1k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ - run: ./lotus fetch-params --proving-params 1024 - save_cache: name: Save parameters cache - key: 'v20-1k-lotus-params-{{ checksum "build/proof-params/parameters.json" }}-{{ checksum "build/paramfetch.go" }}' + key: 'v20-1k-lotus-params' paths: - /var/tmp/filecoin-proof-parameters/ diff --git a/api/api_storage.go b/api/api_storage.go index 1c89f6688..8d0c39038 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/lib/sectorbuilder" + "github.com/filecoin-project/go-sectorbuilder" ) // alias because cbor-gen doesn't like non-alias types diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 90fd41d13..7a95f318c 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -3,7 +3,7 @@ package apistruct import ( "context" - "github.com/filecoin-project/lotus/lib/sectorbuilder" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" diff --git a/build/genesis.go b/build/genesis.go index 29575dd67..cdcf0402e 100644 --- a/build/genesis.go +++ b/build/genesis.go @@ -1,6 +1,12 @@ package build -import rice "github.com/GeertJohan/go.rice" +import ( + rice "github.com/GeertJohan/go.rice" + logging "github.com/ipfs/go-log" +) + +// moved from now-defunct build/paramfetch.go +var log = logging.Logger("build") func MaybeGenesis() []byte { builtinGen, err := rice.FindBox("genesis") diff --git a/build/paramfetch.go b/build/paramfetch.go deleted file mode 100644 index 14bb80f29..000000000 --- a/build/paramfetch.go +++ /dev/null @@ -1,189 +0,0 @@ -package build - -import ( - "encoding/hex" - "encoding/json" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - - rice "github.com/GeertJohan/go.rice" - logging "github.com/ipfs/go-log" - "github.com/minio/blake2b-simd" - "go.uber.org/multierr" - "golang.org/x/xerrors" - pb "gopkg.in/cheggaaa/pb.v1" -) - -var log = logging.Logger("build") - -//const gateway = "http://198.211.99.118/ipfs/" -const gateway = "https://ipfs.io/ipfs/" -const paramdir = "/var/tmp/filecoin-proof-parameters" -const dirEnv = "FIL_PROOFS_PARAMETER_CACHE" - -type paramFile struct { - Cid string `json:"cid"` - Digest string `json:"digest"` - SectorSize uint64 `json:"sector_size"` -} - -type fetch struct { - wg sync.WaitGroup - fetchLk sync.Mutex - - errs []error -} - -func getParamDir() string { - if os.Getenv(dirEnv) == "" { - return paramdir - } - return os.Getenv(dirEnv) -} - -func GetParams(storageSize uint64) error { - if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) { - return err - } - - var params map[string]paramFile - - paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json") - if err := json.Unmarshal(paramBytes, ¶ms); err != nil { - return err - } - - ft := &fetch{} - - for name, info := range params { - if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") { - continue - } - - ft.maybeFetchAsync(name, info) - } - - return ft.wait() -} - -func (ft *fetch) maybeFetchAsync(name string, info paramFile) { - ft.wg.Add(1) - - go func() { - defer ft.wg.Done() - - path := filepath.Join(getParamDir(), name) - - err := ft.checkFile(path, info) - if !os.IsNotExist(err) && err != nil { - log.Warn(err) - } - if err == nil { - return - } - - ft.fetchLk.Lock() - defer ft.fetchLk.Unlock() - - if err := doFetch(path, info); err != nil { - ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err)) - return - } - err = ft.checkFile(path, info) - if err != nil { - ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err)) - err := os.Remove(path) - if err != nil { - ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err)) - } - } - }() -} - -func (ft *fetch) checkFile(path string, info paramFile) error { - if os.Getenv("TRUST_PARAMS") == "1" { - log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION") - return nil - } - - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - - h := blake2b.New512() - if _, err := io.Copy(h, f); err != nil { - return err - } - - sum := h.Sum(nil) - strSum := hex.EncodeToString(sum[:16]) - if strSum == info.Digest { - log.Infof("Parameter file %s is ok", path) - return nil - } - - return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest) -} - -func (ft *fetch) wait() error { - ft.wg.Wait() - return multierr.Combine(ft.errs...) -} - -func doFetch(out string, info paramFile) error { - gw := os.Getenv("IPFS_GATEWAY") - if gw == "" { - gw = gateway - } - log.Infof("Fetching %s from %s", out, gw) - - outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return err - } - defer outf.Close() - - fStat, err := outf.Stat() - if err != nil { - return err - } - header := http.Header{} - header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-") - url, err := url.Parse(gw + info.Cid) - if err != nil { - return err - } - log.Infof("GET %s", url) - - req := http.Request{ - Method: "GET", - URL: url, - Header: header, - Close: true, - } - - resp, err := http.DefaultClient.Do(&req) - if err != nil { - return err - } - defer resp.Body.Close() - - bar := pb.New64(resp.ContentLength) - bar.Units = pb.U_BYTES - bar.ShowSpeed = true - bar.Start() - - _, err = io.Copy(outf, bar.NewProxyReader(resp.Body)) - - bar.Finish() - - return err -} diff --git a/build/params_shared.go b/build/params_shared.go index 302629e5b..ec4982cc1 100644 --- a/build/params_shared.go +++ b/build/params_shared.go @@ -56,11 +56,6 @@ const SealRandomnessLookback = Finality // Epochs const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000 -// 1 / n -const SectorChallengeRatioDiv = 25 - -const MaxFallbackPostChallengeCount = 10 - // ///// // Mining diff --git a/build/proof-params/mkparamfetch.sh b/build/proof-params/mkparamfetch.sh deleted file mode 100755 index a9f8023ea..000000000 --- a/build/proof-params/mkparamfetch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -function b64() { - f="$1" - case `uname` in - Darwin) - base64 -i "$f" - ;; - Linux) - base64 "$f" -w 0 - ;; - esac - printf "unsupported system" 1>&2 - exit 1 -} - -sed "s/{{PARAMSJSON}}/$(b64 build/proof-params/parameters.json)/g" build/proof-params/paramfetch.sh.template > ./build/paramfetch.sh -chmod +x ./build/paramfetch.sh diff --git a/build/proof-params/parameters.json b/build/proof-params/parameters.json deleted file mode 100644 index 7e1988a41..000000000 --- a/build/proof-params/parameters.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": { - "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9", - "digest": "39a9edec3355516674f0d12b926be493", - "sector_size": 34359738368 - }, - "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": { - "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX", - "digest": "0227ae7df4f2affe529ebafbbc7540ee", - "sector_size": 34359738368 - }, - "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": { - "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv", - "digest": "991041a354b12c280542741f58c7f2ca", - "sector_size": 1024 - }, - "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": { - "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU", - "digest": "91fac550e1f9bccab213830bb0c85bd6", - "sector_size": 1024 - }, - "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": { - "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4", - "digest": "6322eacd2773163ddd51f9ca7d645fc4", - "sector_size": 1073741824 - }, - "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": { - "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf", - "digest": "0df62745fceac922e3e70847cfc70b52", - "sector_size": 1073741824 - }, - "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": { - "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9", - "digest": "de9d71e672f286706a1673bd57abdaac", - "sector_size": 16777216 - }, - "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": { - "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu", - "digest": "7f796d3a0f13499181e44b5eee0cc744", - "sector_size": 16777216 - }, - "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": { - "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN", - "digest": "151ae0ae183fc141e8c2bebc28e5cc10", - "sector_size": 268435456 - }, - "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": { - "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji", - "digest": "95eb89588e9d1832aca044c3a13178af", - "sector_size": 268435456 - }, - "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": { - "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX", - "digest": "fcd50e2e08a8560a6bb3418e883567ed", - "sector_size": 268435456 - }, - "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": { - "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y", - "digest": "872e244d86499fd659082e3bcf3f13e7", - "sector_size": 268435456 - }, - "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": { - "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN", - "digest": "0a338b941c5f17946340de5fc95cab30", - "sector_size": 34359738368 - }, - "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": { - "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF", - "digest": "c752e070a6b7aa8b79aa661a6b600b55", - "sector_size": 34359738368 - }, - "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": { - "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186", - "digest": "efa407fd09202dffd15799a8518e73d3", - "sector_size": 1024 - }, - "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": { - "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG", - "digest": "b2f09a0ccb62da28c890d5b881c8dcd2", - "sector_size": 1024 - }, - "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": { - "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q", - "digest": "b53c1916a63839ec345aa2224e9198b7", - "sector_size": 1073741824 - }, - "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": { - "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA", - "digest": "20d58a3fae7343481f8298a2dd493dd7", - "sector_size": 1073741824 - }, - "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": { - "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU", - "digest": "7698426202c7e07b26ef056d31485b3a", - "sector_size": 16777216 - }, - "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": { - "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax", - "digest": "834408e5c3fce6ec5d1bf64e64cee94e", - "sector_size": 16777216 - } -} - diff --git a/build/proof-params/paramfetch.sh.template b/build/proof-params/paramfetch.sh.template deleted file mode 100755 index 739b8b0e1..000000000 --- a/build/proof-params/paramfetch.sh.template +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -PARAMS='{{PARAMSJSON}}' - -die() { - echo "$@" >&2 - exit 1 -} - -have_binary() { - type "$1" > /dev/null 2> /dev/null -} - -check_writable() { - printf "" > "$1" && rm "$1" -} - -try_download() { - url="$1" - output="$2" - command="$3" - util_name="$(set -- $command; echo "$1")" - - if ! have_binary "$util_name"; then - return 1 - fi - - printf '==> Using %s to download "%s" to "%s"\n' "$util_name" "$url" "$output" - if eval "$command"; then - echo "==> Download complete!" - return - else - echo "error: couldn't download with $util_name ($?)" - return 1 - fi -} - -download() { - dl_url="$1" - dl_output="$2" - - test "$#" -eq "2" || die "download requires exactly two arguments, was given $@" - - if ! check_writable "$dl_output"; then - die "download error: cannot write to $dl_output" - fi - - try_download "$dl_url" "$dl_output" "wget '$dl_url' -O '$dl_output'" && return - try_download "$dl_url" "$dl_output" "curl --silent --fail --output '$dl_output' '$dl_url'" && return - try_download "$dl_url" "$dl_output" "fetch '$dl_url' -o '$dl_output'" && return - try_download "$dl_url" "$dl_output" "http '$dl_url' > '$dl_output'" && return - try_download "$dl_url" "$dl_output" "ftp -o '$dl_output' '$dl_url'" && return - - die "Unable to download $dl_url. exiting." -} - -fetch_ipget() { - local dest="$1" - local cid="$2" - - IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i" - - ./bin/ipget $IPGET_PARAMS -o "$dest" "$cid" -} - -fetch_gateway() { - local dest="$1" - local cid="$2" - - local url="http://198.211.99.118/ipfs/$cid" - - download "$url" "$dest" -} - -OUT_DIR="/var/tmp/filecoin-proof-parameters" - -mkdir -p $OUT_DIR -printf $PARAMS | base64 -d | jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output | \ - while read -r dest cid digest; do - if [[ -f "$dest" ]]; then - b2=$(b2sum "$dest" | head -c 32) - if [[ "$digest" == "$b2" ]]; then - echo "$dest exists and has correct hash" - continue - else - echo "$dest has incorrect hash" - rm -f "$dest" - fi - fi - echo "downloading $dest" - - fetch_gateway "$dest" "$cid" - done diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 19b319662..54076cac8 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -9,10 +9,10 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/go-amt-ipld" "github.com/ipfs/go-cid" diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go index 173e7e6e0..a522fe349 100644 --- a/chain/actors/actor_miner_test.go +++ b/chain/actors/actor_miner_test.go @@ -7,12 +7,12 @@ import ( "testing" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" hamt "github.com/ipfs/go-hamt-ipld" blockstore "github.com/ipfs/go-ipfs-blockstore" cbg "github.com/whyrusleeping/cbor-gen" diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 526099e00..e4ad58cb5 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -14,10 +14,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type StorageMarketActor struct{} diff --git a/chain/deals/client.go b/chain/deals/client.go index afbb317f0..709432f63 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -12,6 +12,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/events" @@ -20,7 +21,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/retrieval/discovery" diff --git a/chain/deals/client_utils.go b/chain/deals/client_utils.go index 256738366..944846ed0 100644 --- a/chain/deals/client_utils.go +++ b/chain/deals/client_utils.go @@ -5,6 +5,7 @@ import ( "context" "runtime" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" unixfile "github.com/ipfs/go-unixfs/file" @@ -12,11 +13,11 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-statestore" + "github.com/filecoin-project/lotus/datatransfer" "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/chain/deals/provider.go b/chain/deals/provider.go index a12f9fa07..9e45ef8b2 100644 --- a/chain/deals/provider.go +++ b/chain/deals/provider.go @@ -14,11 +14,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sectorblocks" diff --git a/chain/deals/provider_utils.go b/chain/deals/provider_utils.go index c3e77b760..e1fcfcf32 100644 --- a/chain/deals/provider_utils.go +++ b/chain/deals/provider_utils.go @@ -12,9 +12,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/ipfs/go-cid" inet "github.com/libp2p/go-libp2p-core/network" diff --git a/chain/deals/request_validation_test.go b/chain/deals/request_validation_test.go index c0bb34e03..6da68d219 100644 --- a/chain/deals/request_validation_test.go +++ b/chain/deals/request_validation_test.go @@ -15,11 +15,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/statestore" ) var blockGenerator = blocksutil.NewBlockGenerator() diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 20b238dd8..d476a0937 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -11,6 +11,7 @@ import ( ffi "github.com/filecoin-project/filecoin-ffi" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-car" offline "github.com/ipfs/go-ipfs-exchange-offline" @@ -28,7 +29,6 @@ import ( "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/repo" block "github.com/ipfs/go-block-format" diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index c9bcd8529..109848f8e 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -4,12 +4,13 @@ import ( "context" ffi "github.com/filecoin-project/filecoin-ffi" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" amt "github.com/filecoin-project/go-amt-ipld" cid "github.com/ipfs/go-cid" diff --git a/chain/sync.go b/chain/sync.go index c6907b848..19d83df27 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -12,6 +12,7 @@ import ( "github.com/Gurpartap/async" bls "github.com/filecoin-project/filecoin-ffi" amt "github.com/filecoin-project/go-amt-ipld" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/hashicorp/go-multierror" "github.com/ipfs/go-cid" dstore "github.com/ipfs/go-datastore" @@ -26,6 +27,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -35,7 +37,6 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) var log = logging.Logger("chain") diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index adbea8951..23fdab0f7 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -5,6 +5,8 @@ import ( "context" "math/big" + "github.com/filecoin-project/go-sectorbuilder" + block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/minio/sha256-simd" @@ -13,6 +15,7 @@ import ( xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/build" ) @@ -214,11 +217,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big } func ElectionPostChallengeCount(sectors uint64, faults int) uint64 { - if sectors == 0 { - return 0 - } - // ceil(sectors / build.SectorChallengeRatioDiv) - return (sectors-uint64(faults)-1)/build.SectorChallengeRatioDiv + 1 + return sectorbuilder.ElectionPostChallengeCount(sectors, faults) } func (t *Ticket) Equals(ot *Ticket) bool { diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index b1ff4b838..57b963041 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -23,10 +23,10 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) var log = logging.Logger("lotus-bench") diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index ec514b0e9..fa1b00a8a 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -7,8 +7,8 @@ import ( paramfetch "github.com/filecoin-project/go-paramfetch" "golang.org/x/xerrors" + "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type worker struct { diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go index af4681a1f..9697a4bad 100644 --- a/cmd/lotus-seal-worker/transfer.go +++ b/cmd/lotus-seal-worker/transfer.go @@ -6,12 +6,12 @@ import ( "net/http" "os" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" files "github.com/ipfs/go-ipfs-files" "golang.org/x/xerrors" "gopkg.in/cheggaaa/pb.v1" "path/filepath" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/lib/tarutil" ) diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index f79655294..ee1481d3b 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -7,6 +7,7 @@ import ( "encoding/json" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" "github.com/mitchellh/go-homedir" @@ -17,7 +18,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) var log = logging.Logger("lotus-seed") diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 520748b82..e98abdb0b 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" "golang.org/x/xerrors" @@ -20,7 +21,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) var log = logging.Logger("preseal") diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 0f2a83d39..9bf52fea8 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/go-sectorbuilder" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -29,7 +30,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" diff --git a/go.mod b/go.mod index c0bf799d6..3283f456b 100644 --- a/go.mod +++ b/go.mod @@ -9,18 +9,18 @@ require ( github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/docker/go-units v0.4.0 - github.com/fatih/color v1.7.0 // indirect github.com/filecoin-project/chain-validation v0.0.3 - github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed + github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590 github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 + github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 + github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/go-cmp v0.3.1 // indirect - github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.1 github.com/hashicorp/go-multierror v1.0.0 @@ -69,12 +69,10 @@ require ( github.com/libp2p/go-libp2p-tls v0.1.0 github.com/libp2p/go-libp2p-yamux v0.2.1 github.com/libp2p/go-maddr-filter v0.0.5 - github.com/mattn/go-isatty v0.0.9 // indirect github.com/mattn/go-sqlite3 v1.12.0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 - github.com/mr-tron/base58 v1.1.3 // indirect github.com/multiformats/go-base32 v0.0.3 github.com/multiformats/go-multiaddr v0.1.1 github.com/multiformats/go-multiaddr-dns v0.2.0 @@ -84,24 +82,19 @@ require ( github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/otiai10/copy v1.0.2 github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a - github.com/smartystreets/assertions v1.0.1 // indirect - github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d - go.opencensus.io v0.22.1 + go.opencensus.io v0.22.2 go.uber.org/dig v1.7.0 // indirect go.uber.org/fx v1.9.0 go.uber.org/goleak v0.10.0 // indirect go.uber.org/multierr v1.4.0 go.uber.org/zap v1.10.0 go4.org v0.0.0-20190313082347-94abd6928b1d // indirect - golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect - golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/cheggaaa/pb.v1 v1.0.28 diff --git a/go.sum b/go.sum index 3f09e3afb..34d5667dc 100644 --- a/go.sum +++ b/go.sum @@ -85,12 +85,16 @@ github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2j github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cODZD2YzpTUtrOSxbEnWFcQHidNRZiRdvLxySjGvG/M= github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= -github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= -github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY= +github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc= +github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk= +github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= @@ -179,6 +183,7 @@ github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -530,6 +535,7 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= @@ -660,6 +666,8 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= diff --git a/lib/sectorbuilder/files.go b/lib/sectorbuilder/files.go deleted file mode 100644 index a8c5dd133..000000000 --- a/lib/sectorbuilder/files.go +++ /dev/null @@ -1,126 +0,0 @@ -package sectorbuilder - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - - "golang.org/x/xerrors" -) - -func (sb *SectorBuilder) SectorName(sectorID uint64) string { - return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID) -} - -func (sb *SectorBuilder) StagedSectorPath(sectorID uint64) string { - return filepath.Join(sb.filesystem.pathFor(dataStaging), sb.SectorName(sectorID)) -} - -func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string { - return filepath.Join(sb.filesystem.pathFor(dataUnsealed), sb.SectorName(sectorID)) -} - -func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) { - return os.OpenFile(sb.StagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644) -} - -func (sb *SectorBuilder) SealedSectorPath(sectorID uint64) (string, error) { - path := filepath.Join(sb.filesystem.pathFor(dataSealed), sb.SectorName(sectorID)) - - return path, nil -} - -func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) { - dir := filepath.Join(sb.filesystem.pathFor(dataCache), sb.SectorName(sectorID)) - - err := os.Mkdir(dir, 0755) - if os.IsExist(err) { - err = nil - } - - return dir, err -} - -func (sb *SectorBuilder) GetPath(typ string, sectorName string) (string, error) { - _, found := overheadMul[dataType(typ)] - if !found { - return "", xerrors.Errorf("unknown sector type: %s", typ) - } - - return filepath.Join(sb.filesystem.pathFor(dataType(typ)), sectorName), nil -} - -func (sb *SectorBuilder) TrimCache(sectorID uint64) error { - dir, err := sb.sectorCacheDir(sectorID) - if err != nil { - return xerrors.Errorf("getting cache dir: %w", err) - } - - files, err := ioutil.ReadDir(dir) - if err != nil { - return xerrors.Errorf("readdir: %w", err) - } - - for _, file := range files { - if !strings.HasSuffix(file.Name(), ".dat") { // _aux probably - continue - } - if strings.HasSuffix(file.Name(), "-data-tree-r-last.dat") { // Want to keep - continue - } - - if err := os.Remove(filepath.Join(dir, file.Name())); err != nil { - return xerrors.Errorf("rm %s: %w", file.Name(), err) - } - } - - return nil -} - -func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { - f, ok := r.(*os.File) - if ok { - return f, func() error { return nil }, nil - } - - var w *os.File - - f, w, err := os.Pipe() - if err != nil { - return nil, nil, err - } - - var wait sync.Mutex - var werr error - - wait.Lock() - go func() { - defer wait.Unlock() - - var copied int64 - copied, werr = io.CopyN(w, r, n) - if werr != nil { - log.Warnf("toReadableFile: copy error: %+v", werr) - } - - err := w.Close() - if werr == nil && err != nil { - werr = err - log.Warnf("toReadableFile: close error: %+v", err) - return - } - if copied != n { - log.Warnf("copied different amount than expected: %d != %d", copied, n) - werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) - } - }() - - return f, func() error { - wait.Lock() - return werr - }, nil -} diff --git a/lib/sectorbuilder/fs.go b/lib/sectorbuilder/fs.go deleted file mode 100644 index e427e8014..000000000 --- a/lib/sectorbuilder/fs.go +++ /dev/null @@ -1,116 +0,0 @@ -package sectorbuilder - -import ( - "github.com/filecoin-project/lotus/chain/types" - "golang.org/x/xerrors" - "os" - "path/filepath" - "sync" - "syscall" -) - -type dataType string - -const ( - dataCache dataType = "cache" - dataStaging dataType = "staging" - dataSealed dataType = "sealed" - dataUnsealed dataType = "unsealed" -) - -var overheadMul = map[dataType]uint64{ // * sectorSize - dataCache: 11, // TODO: check if true for 32G sectors - dataStaging: 1, - dataSealed: 1, - dataUnsealed: 1, -} - -type fs struct { - path string - - // in progress actions - - reserved map[dataType]uint64 - - lk sync.Mutex -} - -func openFs(dir string) *fs { - return &fs{ - path: dir, - reserved: map[dataType]uint64{}, - } -} - -func (f *fs) init() error { - for _, dir := range []string{f.path, - f.pathFor(dataCache), - f.pathFor(dataStaging), - f.pathFor(dataSealed), - f.pathFor(dataUnsealed)} { - if err := os.Mkdir(dir, 0755); err != nil { - if os.IsExist(err) { - continue - } - return err - } - } - - return nil -} - -func (f *fs) pathFor(typ dataType) string { - _, found := overheadMul[typ] - if !found { - panic("unknown data path requested") - } - - return filepath.Join(f.path, string(typ)) -} - -func (f *fs) reservedBytes() int64 { - var out int64 - for _, r := range f.reserved { - out += int64(r) - } - return out -} - -func (f *fs) reserve(typ dataType, size uint64) error { - f.lk.Lock() - defer f.lk.Unlock() - - var fsstat syscall.Statfs_t - - if err := syscall.Statfs(f.pathFor(typ), &fsstat); err != nil { - return err - } - - fsavail := int64(fsstat.Bavail) * int64(fsstat.Bsize) - - avail := fsavail - f.reservedBytes() - - need := overheadMul[typ] * size - - if int64(need) > avail { - return xerrors.Errorf("not enough space in '%s', need %s, available %s (fs: %s, reserved: %s)", - f.path, - types.NewInt(need).SizeStr(), - types.NewInt(uint64(avail)).SizeStr(), - types.NewInt(uint64(fsavail)).SizeStr(), - types.NewInt(uint64(f.reservedBytes())).SizeStr()) - } - - f.reserved[typ] += need - - return nil -} - -func (f *fs) free(typ dataType, sectorSize uint64) { - f.lk.Lock() - defer f.lk.Unlock() - - f.reserved[typ] -= overheadMul[typ] * sectorSize - - return -} diff --git a/lib/sectorbuilder/mock.go b/lib/sectorbuilder/mock.go deleted file mode 100644 index fe90a64f0..000000000 --- a/lib/sectorbuilder/mock.go +++ /dev/null @@ -1,27 +0,0 @@ -package sectorbuilder - -import ( - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) { - addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq") - if err != nil { - return nil, err - } - - sb, err := New(&Config{ - SectorSize: sectorSize, - - Dir: dir, - - WorkerThreads: 2, - Miner: addr, - }, ds) - if err != nil { - return nil, err - } - - return sb, nil -} diff --git a/lib/sectorbuilder/remote.go b/lib/sectorbuilder/remote.go deleted file mode 100644 index 9fa4190ff..000000000 --- a/lib/sectorbuilder/remote.go +++ /dev/null @@ -1,174 +0,0 @@ -package sectorbuilder - -import ( - "context" - - "golang.org/x/xerrors" -) - -type WorkerTaskType int - -const ( - WorkerPreCommit WorkerTaskType = iota - WorkerCommit -) - -type WorkerTask struct { - Type WorkerTaskType - TaskID uint64 - - SectorID uint64 - - // preCommit - SealTicket SealTicket - Pieces []PublicPieceInfo - - // commit - SealSeed SealSeed - Rspco RawSealPreCommitOutput -} - -type workerCall struct { - task WorkerTask - ret chan SealRes -} - -func (sb *SectorBuilder) AddWorker(ctx context.Context, cfg WorkerCfg) (<-chan WorkerTask, error) { - sb.remoteLk.Lock() - defer sb.remoteLk.Unlock() - - taskCh := make(chan WorkerTask) - r := &remote{ - sealTasks: taskCh, - busy: 0, - } - - sb.remoteCtr++ - sb.remotes[sb.remoteCtr] = r - - go sb.remoteWorker(ctx, r, cfg) - - return taskCh, nil -} - -func (sb *SectorBuilder) returnTask(task workerCall) { - var ret chan workerCall - switch task.task.Type { - case WorkerPreCommit: - ret = sb.precommitTasks - case WorkerCommit: - ret = sb.commitTasks - default: - log.Error("unknown task type", task.task.Type) - } - - go func() { - select { - case ret <- task: - case <-sb.stopping: - return - } - }() -} - -func (sb *SectorBuilder) remoteWorker(ctx context.Context, r *remote, cfg WorkerCfg) { - defer log.Warn("Remote worker disconnected") - - defer func() { - sb.remoteLk.Lock() - defer sb.remoteLk.Unlock() - - for i, vr := range sb.remotes { - if vr == r { - delete(sb.remotes, i) - return - } - } - }() - - precommits := sb.precommitTasks - if cfg.NoPreCommit { - precommits = nil - } - commits := sb.commitTasks - if cfg.NoCommit { - commits = nil - } - - for { - select { - case task := <-commits: - sb.doTask(ctx, r, task) - case task := <-precommits: - sb.doTask(ctx, r, task) - case <-ctx.Done(): - return - case <-sb.stopping: - return - } - - r.lk.Lock() - r.busy = 0 - r.lk.Unlock() - } -} - -func (sb *SectorBuilder) doTask(ctx context.Context, r *remote, task workerCall) { - resCh := make(chan SealRes) - - sb.remoteLk.Lock() - sb.remoteResults[task.task.TaskID] = resCh - sb.remoteLk.Unlock() - - // send the task - select { - case r.sealTasks <- task.task: - case <-ctx.Done(): - sb.returnTask(task) - return - } - - r.lk.Lock() - r.busy = task.task.TaskID - r.lk.Unlock() - - // wait for the result - select { - case res := <-resCh: - - // send the result back to the caller - select { - case task.ret <- res: - case <-ctx.Done(): - return - case <-sb.stopping: - return - } - - case <-ctx.Done(): - log.Warnf("context expired while waiting for task %d (sector %d): %s", task.task.TaskID, task.task.SectorID, ctx.Err()) - return - case <-sb.stopping: - return - } -} - -func (sb *SectorBuilder) TaskDone(ctx context.Context, task uint64, res SealRes) error { - sb.remoteLk.Lock() - rres, ok := sb.remoteResults[task] - if ok { - delete(sb.remoteResults, task) - } - sb.remoteLk.Unlock() - - if !ok { - return xerrors.Errorf("task %d not found", task) - } - - select { - case rres <- res: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} diff --git a/lib/sectorbuilder/scrub.go b/lib/sectorbuilder/scrub.go deleted file mode 100644 index a6ed8ce7c..000000000 --- a/lib/sectorbuilder/scrub.go +++ /dev/null @@ -1,84 +0,0 @@ -package sectorbuilder - -import ( - "io/ioutil" - "os" - "path/filepath" - - sectorbuilder "github.com/filecoin-project/filecoin-ffi" - "golang.org/x/xerrors" -) - -type Fault struct { - SectorID uint64 - - Err error -} - -func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault { - var faults []*Fault - - for _, sector := range sectorSet.Values() { - err := sb.checkSector(sector.SectorID) - if err != nil { - faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err}) - } - } - - return faults -} - -func (sb *SectorBuilder) checkSector(sectorID uint64) error { - cache, err := sb.sectorCacheDir(sectorID) - if err != nil { - return xerrors.Errorf("getting sector cache dir: %w", err) - } - - if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil { - return err - } - if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil { - return err - } - - // TODO: better validate this - if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be? - return err - } - - dent, err := ioutil.ReadDir(cache) - if err != nil { - return xerrors.Errorf("reading cache dir %s", cache) - } - if len(dent) != 3 { - return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache) - } - - sealed, err := sb.SealedSectorPath(sectorID) - if err != nil { - return xerrors.Errorf("getting sealed sector path: %w", err) - } - - if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil { - return err - } - - return nil -} - -func assertFile(path string, minSz uint64, maxSz uint64) error { - st, err := os.Stat(path) - if err != nil { - return xerrors.Errorf("stat %s: %w", path, err) - } - - if st.IsDir() { - return xerrors.Errorf("expected %s to be a regular file", path) - } - - if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz { - return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size()) - } - - return nil -} diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go deleted file mode 100644 index 206ea41b2..000000000 --- a/lib/sectorbuilder/sectorbuilder.go +++ /dev/null @@ -1,799 +0,0 @@ -package sectorbuilder - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "sync" - "sync/atomic" - - sectorbuilder "github.com/filecoin-project/filecoin-ffi" - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log" - dcopy "github.com/otiai10/copy" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/node/modules/dtypes" -) - -const PoStReservedWorkers = 1 -const PoRepProofPartitions = 10 - -var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last") - -var log = logging.Logger("sectorbuilder") - -type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo -type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo - -type SealTicket = sectorbuilder.SealTicket - -type SealSeed = sectorbuilder.SealSeed - -type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput - -type SealCommitOutput = sectorbuilder.SealCommitOutput - -type PublicPieceInfo = sectorbuilder.PublicPieceInfo - -type RawSealPreCommitOutput sectorbuilder.RawSealPreCommitOutput - -type EPostCandidate = sectorbuilder.Candidate - -const CommLen = sectorbuilder.CommitmentBytesLen - -type WorkerCfg struct { - NoPreCommit bool - NoCommit bool - - // TODO: 'cost' info, probably in terms of sealing + transfer speed -} - -type SectorBuilder struct { - ds dtypes.MetadataDS - idLk sync.Mutex - - ssize uint64 - lastID uint64 - - Miner address.Address - - unsealLk sync.Mutex - - noCommit bool - noPreCommit bool - rateLimit chan struct{} - - precommitTasks chan workerCall - commitTasks chan workerCall - - taskCtr uint64 - remoteLk sync.Mutex - remoteCtr int - remotes map[int]*remote - remoteResults map[uint64]chan<- SealRes - - addPieceWait int32 - preCommitWait int32 - commitWait int32 - unsealWait int32 - - fsLk sync.Mutex - filesystem *fs // TODO: multi-fs support - - stopping chan struct{} -} - -type JsonRSPCO struct { - CommD []byte - CommR []byte -} - -func (rspco *RawSealPreCommitOutput) ToJson() JsonRSPCO { - return JsonRSPCO{ - CommD: rspco.CommD[:], - CommR: rspco.CommR[:], - } -} - -func (rspco *JsonRSPCO) rspco() RawSealPreCommitOutput { - var out RawSealPreCommitOutput - copy(out.CommD[:], rspco.CommD) - copy(out.CommR[:], rspco.CommR) - return out -} - -type SealRes struct { - Err string - GoErr error `json:"-"` - - Proof []byte - Rspco JsonRSPCO -} - -type remote struct { - lk sync.Mutex - - sealTasks chan<- WorkerTask - busy uint64 // only for metrics -} - -type Config struct { - SectorSize uint64 - Miner address.Address - - WorkerThreads uint8 - FallbackLastID uint64 - NoCommit bool - NoPreCommit bool - - Dir string - _ struct{} // guard against nameless init -} - -func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) { - if cfg.WorkerThreads < PoStReservedWorkers { - return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers, cfg.WorkerThreads) - } - - var lastUsedID uint64 - b, err := ds.Get(lastSectorIdKey) - switch err { - case nil: - i, err := strconv.ParseInt(string(b), 10, 64) - if err != nil { - return nil, err - } - lastUsedID = uint64(i) - case datastore.ErrNotFound: - lastUsedID = cfg.FallbackLastID - default: - return nil, err - } - - rlimit := cfg.WorkerThreads - PoStReservedWorkers - - sealLocal := rlimit > 0 - - if rlimit == 0 { - rlimit = 1 - } - - sb := &SectorBuilder{ - ds: ds, - - ssize: cfg.SectorSize, - lastID: lastUsedID, - - filesystem: openFs(cfg.Dir), - - Miner: cfg.Miner, - - noPreCommit: cfg.NoPreCommit || !sealLocal, - noCommit: cfg.NoCommit || !sealLocal, - rateLimit: make(chan struct{}, rlimit), - - taskCtr: 1, - precommitTasks: make(chan workerCall), - commitTasks: make(chan workerCall), - remoteResults: map[uint64]chan<- SealRes{}, - remotes: map[int]*remote{}, - - stopping: make(chan struct{}), - } - - if err := sb.filesystem.init(); err != nil { - return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err) - } - - return sb, nil -} - -func NewStandalone(cfg *Config) (*SectorBuilder, error) { - sb := &SectorBuilder{ - ds: nil, - - ssize: cfg.SectorSize, - - Miner: cfg.Miner, - filesystem: openFs(cfg.Dir), - - taskCtr: 1, - remotes: map[int]*remote{}, - rateLimit: make(chan struct{}, cfg.WorkerThreads), - stopping: make(chan struct{}), - } - - if err := sb.filesystem.init(); err != nil { - return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err) - } - - return sb, nil -} - -func (sb *SectorBuilder) checkRateLimit() { - if cap(sb.rateLimit) == len(sb.rateLimit) { - log.Warn("rate-limiting local sectorbuilder call") - } -} - -func (sb *SectorBuilder) RateLimit() func() { - sb.checkRateLimit() - - sb.rateLimit <- struct{}{} - - return func() { - <-sb.rateLimit - } -} - -type WorkerStats struct { - LocalFree int - LocalReserved int - LocalTotal int - // todo: post in progress - RemotesTotal int - RemotesFree int - - AddPieceWait int - PreCommitWait int - CommitWait int - UnsealWait int -} - -func (sb *SectorBuilder) WorkerStats() WorkerStats { - sb.remoteLk.Lock() - defer sb.remoteLk.Unlock() - - remoteFree := len(sb.remotes) - for _, r := range sb.remotes { - if r.busy > 0 { - remoteFree-- - } - } - - return WorkerStats{ - LocalFree: cap(sb.rateLimit) - len(sb.rateLimit), - LocalReserved: PoStReservedWorkers, - LocalTotal: cap(sb.rateLimit) + PoStReservedWorkers, - RemotesTotal: len(sb.remotes), - RemotesFree: remoteFree, - - AddPieceWait: int(atomic.LoadInt32(&sb.addPieceWait)), - PreCommitWait: int(atomic.LoadInt32(&sb.preCommitWait)), - CommitWait: int(atomic.LoadInt32(&sb.commitWait)), - UnsealWait: int(atomic.LoadInt32(&sb.unsealWait)), - } -} - -func addressToProverID(a address.Address) [32]byte { - var proverId [32]byte - copy(proverId[:], a.Payload()) - return proverId -} - -func (sb *SectorBuilder) AcquireSectorId() (uint64, error) { - sb.idLk.Lock() - defer sb.idLk.Unlock() - - sb.lastID++ - id := sb.lastID - - err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))) - if err != nil { - return 0, err - } - return id, nil -} - -func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) { - fs := sb.filesystem - - if err := fs.reserve(dataStaging, sb.ssize); err != nil { - return PublicPieceInfo{}, err - } - defer fs.free(dataStaging, sb.ssize) - - atomic.AddInt32(&sb.addPieceWait, 1) - ret := sb.RateLimit() - atomic.AddInt32(&sb.addPieceWait, -1) - defer ret() - - f, werr, err := toReadableFile(file, int64(pieceSize)) - if err != nil { - return PublicPieceInfo{}, err - } - - stagedFile, err := sb.stagedSectorFile(sectorId) - if err != nil { - return PublicPieceInfo{}, err - } - - _, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes) - if err != nil { - return PublicPieceInfo{}, err - } - - if err := stagedFile.Close(); err != nil { - return PublicPieceInfo{}, err - } - - if err := f.Close(); err != nil { - return PublicPieceInfo{}, err - } - - return PublicPieceInfo{ - Size: pieceSize, - CommP: commP, - }, werr() -} - -func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) { - fs := sb.filesystem - - if err := fs.reserve(dataUnsealed, sb.ssize); err != nil { // TODO: this needs to get smarter when we start supporting partial unseals - return nil, err - } - defer fs.free(dataUnsealed, sb.ssize) - - atomic.AddInt32(&sb.unsealWait, 1) - // TODO: Don't wait if cached - ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker - defer ret() - atomic.AddInt32(&sb.unsealWait, -1) - - sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel - defer sb.unsealLk.Unlock() - - cacheDir, err := sb.sectorCacheDir(sectorID) - if err != nil { - return nil, err - } - - sealedPath, err := sb.SealedSectorPath(sectorID) - if err != nil { - return nil, err - } - - unsealedPath := sb.unsealedSectorPath(sectorID) - - // TODO: GC for those - // (Probably configurable count of sectors to be kept unsealed, and just - // remove last used one (or use whatever other cache policy makes sense)) - f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644) - if err != nil { - if !os.IsNotExist(err) { - return nil, err - } - - var commd [CommLen]byte - copy(commd[:], commD) - - var tkt [CommLen]byte - copy(tkt[:], ticket) - - err = sectorbuilder.Unseal(sb.ssize, - PoRepProofPartitions, - cacheDir, - sealedPath, - unsealedPath, - sectorID, - addressToProverID(sb.Miner), - tkt, - commd) - if err != nil { - return nil, xerrors.Errorf("unseal failed: %w", err) - } - - f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644) - if err != nil { - return nil, err - } - } - - if _, err := f.Seek(int64(offset), io.SeekStart); err != nil { - return nil, xerrors.Errorf("seek: %w", err) - } - - lr := io.LimitReader(f, int64(size)) - - return &struct { - io.Reader - io.Closer - }{ - Reader: lr, - Closer: f, - }, nil -} - -func (sb *SectorBuilder) sealPreCommitRemote(call workerCall) (RawSealPreCommitOutput, error) { - atomic.AddInt32(&sb.preCommitWait, -1) - - select { - case ret := <-call.ret: - var err error - if ret.Err != "" { - err = xerrors.New(ret.Err) - } - return ret.Rspco.rspco(), err - case <-sb.stopping: - return RawSealPreCommitOutput{}, xerrors.New("sectorbuilder stopped") - } -} - -func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) { - fs := sb.filesystem - - if err := fs.reserve(dataCache, sb.ssize); err != nil { - return RawSealPreCommitOutput{}, err - } - defer fs.free(dataCache, sb.ssize) - - if err := fs.reserve(dataSealed, sb.ssize); err != nil { - return RawSealPreCommitOutput{}, err - } - defer fs.free(dataSealed, sb.ssize) - - call := workerCall{ - task: WorkerTask{ - Type: WorkerPreCommit, - TaskID: atomic.AddUint64(&sb.taskCtr, 1), - SectorID: sectorID, - SealTicket: ticket, - Pieces: pieces, - }, - ret: make(chan SealRes), - } - - atomic.AddInt32(&sb.preCommitWait, 1) - - select { // prefer remote - case sb.precommitTasks <- call: - return sb.sealPreCommitRemote(call) - default: - } - - sb.checkRateLimit() - - rl := sb.rateLimit - if sb.noPreCommit { - rl = make(chan struct{}) - } - - select { // use whichever is available - case sb.precommitTasks <- call: - return sb.sealPreCommitRemote(call) - case rl <- struct{}{}: - } - - atomic.AddInt32(&sb.preCommitWait, -1) - - // local - - defer func() { - <-sb.rateLimit - }() - - cacheDir, err := sb.sectorCacheDir(sectorID) - if err != nil { - return RawSealPreCommitOutput{}, xerrors.Errorf("getting cache dir: %w", err) - } - - sealedPath, err := sb.SealedSectorPath(sectorID) - if err != nil { - return RawSealPreCommitOutput{}, xerrors.Errorf("getting sealed sector path: %w", err) - } - - e, err := os.OpenFile(sealedPath, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return RawSealPreCommitOutput{}, xerrors.Errorf("ensuring sealed file exists: %w", err) - } - if err := e.Close(); err != nil { - return RawSealPreCommitOutput{}, err - } - - var sum uint64 - for _, piece := range pieces { - sum += piece.Size - } - ussize := UserBytesForSectorSize(sb.ssize) - if sum != ussize { - return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum)) - } - - stagedPath := sb.StagedSectorPath(sectorID) - - rspco, err := sectorbuilder.SealPreCommit( - sb.ssize, - PoRepProofPartitions, - cacheDir, - stagedPath, - sealedPath, - sectorID, - addressToProverID(sb.Miner), - ticket.TicketBytes, - pieces, - ) - if err != nil { - return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err) - } - - return RawSealPreCommitOutput(rspco), nil -} - -func (sb *SectorBuilder) sealCommitRemote(call workerCall) (proof []byte, err error) { - atomic.AddInt32(&sb.commitWait, -1) - - select { - case ret := <-call.ret: - if ret.Err != "" { - err = xerrors.New(ret.Err) - } - return ret.Proof, err - case <-sb.stopping: - return nil, xerrors.New("sectorbuilder stopped") - } -} - -func (sb *SectorBuilder) sealCommitLocal(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) { - atomic.AddInt32(&sb.commitWait, -1) - - defer func() { - <-sb.rateLimit - }() - - cacheDir, err := sb.sectorCacheDir(sectorID) - if err != nil { - return nil, err - } - - proof, err = sectorbuilder.SealCommit( - sb.ssize, - PoRepProofPartitions, - cacheDir, - sectorID, - addressToProverID(sb.Miner), - ticket.TicketBytes, - seed.TicketBytes, - pieces, - sectorbuilder.RawSealPreCommitOutput(rspco), - ) - if err != nil { - log.Warn("StandaloneSealCommit error: ", err) - log.Warnf("sid:%d tkt:%v seed:%v, ppi:%v rspco:%v", sectorID, ticket, seed, pieces, rspco) - - return nil, xerrors.Errorf("StandaloneSealCommit: %w", err) - } - - return proof, nil -} - -func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) { - call := workerCall{ - task: WorkerTask{ - Type: WorkerCommit, - TaskID: atomic.AddUint64(&sb.taskCtr, 1), - SectorID: sectorID, - SealTicket: ticket, - Pieces: pieces, - - SealSeed: seed, - Rspco: rspco, - }, - ret: make(chan SealRes), - } - - atomic.AddInt32(&sb.commitWait, 1) - - select { // prefer remote - case sb.commitTasks <- call: - proof, err = sb.sealCommitRemote(call) - default: - sb.checkRateLimit() - - rl := sb.rateLimit - if sb.noCommit { - rl = make(chan struct{}) - } - - select { // use whichever is available - case sb.commitTasks <- call: - proof, err = sb.sealCommitRemote(call) - case rl <- struct{}{}: - proof, err = sb.sealCommitLocal(sectorID, ticket, seed, pieces, rspco) - } - } - if err != nil { - return nil, xerrors.Errorf("commit: %w", err) - } - - return proof, nil -} - -func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) { - if len(challengeSeed) != CommLen { - return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen) - } - var cseed [CommLen]byte - copy(cseed[:], challengeSeed) - - privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults - if err != nil { - return nil, err - } - - proverID := addressToProverID(sb.Miner) - - return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners) -} - -func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) { - privsectors, err := sb.pubSectorToPriv(sectorInfo, faults) - if err != nil { - return nil, err - } - - challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults)) - - proverID := addressToProverID(sb.Miner) - return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) -} - -func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) { - fmap := map[uint64]struct{}{} - for _, fault := range faults { - fmap[fault] = struct{}{} - } - - var out []sectorbuilder.PrivateSectorInfo - for _, s := range sectorInfo.Values() { - if _, faulty := fmap[s.SectorID]; faulty { - continue - } - - cachePath, err := sb.sectorCacheDir(s.SectorID) - if err != nil { - return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err) - } - - sealedPath, err := sb.SealedSectorPath(s.SectorID) - if err != nil { - return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err) - } - - out = append(out, sectorbuilder.PrivateSectorInfo{ - SectorID: s.SectorID, - CommR: s.CommR, - CacheDirPath: cachePath, - SealedSectorPath: sealedPath, - }) - } - return NewSortedPrivateSectorInfo(out), nil -} - -func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) { - privsectors, err := sb.pubSectorToPriv(sectorInfo, faults) - if err != nil { - return nil, nil, err - } - - challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults)) - - proverID := addressToProverID(sb.Miner) - candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) - if err != nil { - return nil, nil, err - } - - proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates) - return candidates, proof, err -} - -func (sb *SectorBuilder) Stop() { - close(sb.stopping) -} - -func fallbackPostChallengeCount(sectors uint64, faults int) uint64 { - challengeCount := types.ElectionPostChallengeCount(sectors, faults) - if challengeCount > build.MaxFallbackPostChallengeCount { - return build.MaxFallbackPostChallengeCount - } - return challengeCount -} - -func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error { - if err := migrate(osb.filesystem.pathFor(dataCache), sb.filesystem.pathFor(dataCache), symlink); err != nil { - return err - } - - if err := migrate(osb.filesystem.pathFor(dataStaging), sb.filesystem.pathFor(dataStaging), symlink); err != nil { - return err - } - - if err := migrate(osb.filesystem.pathFor(dataSealed), sb.filesystem.pathFor(dataSealed), symlink); err != nil { - return err - } - - val, err := osb.ds.Get(lastSectorIdKey) - if err != nil { - return err - } - - if err := sb.ds.Put(lastSectorIdKey, val); err != nil { - return err - } - - sb.lastID = osb.lastID - - return nil -} - -func (sb *SectorBuilder) SetLastSectorID(id uint64) error { - if err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))); err != nil { - return err - } - - sb.lastID = id - return nil -} - -func migrate(from, to string, symlink bool) error { - st, err := os.Stat(from) - if err != nil { - return err - } - - if st.IsDir() { - return migrateDir(from, to, symlink) - } - return migrateFile(from, to, symlink) -} - -func migrateDir(from, to string, symlink bool) error { - tost, err := os.Stat(to) - if err != nil { - if !os.IsNotExist(err) { - return err - } - - if err := os.MkdirAll(to, 0755); err != nil { - return err - } - } else if !tost.IsDir() { - return xerrors.Errorf("target %q already exists and is a file (expected directory)") - } - - dirents, err := ioutil.ReadDir(from) - if err != nil { - return err - } - - for _, inf := range dirents { - n := inf.Name() - if inf.IsDir() { - if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil { - return err - } - } else { - if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil { - return err - } - } - } - - return nil -} - -func migrateFile(from, to string, symlink bool) error { - if symlink { - return os.Symlink(from, to) - } - - return dcopy.Copy(from, to) -} diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go deleted file mode 100644 index 7720c59b1..000000000 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ /dev/null @@ -1,365 +0,0 @@ -package sectorbuilder_test - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "math/rand" - "os" - "runtime" - "sync" - "testing" - "time" - - ffi "github.com/filecoin-project/filecoin-ffi" - paramfetch "github.com/filecoin-project/go-paramfetch" - "github.com/ipfs/go-datastore" - logging "github.com/ipfs/go-log" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/lib/sectorbuilder" -) - -func init() { - logging.SetLogLevel("*", "INFO") -} - -const sectorSize = 1024 - -type seal struct { - sid uint64 - - pco sectorbuilder.RawSealPreCommitOutput - ppi sectorbuilder.PublicPieceInfo - - ticket sectorbuilder.SealTicket -} - -func (s *seal) precommit(t *testing.T, sb *sectorbuilder.SectorBuilder, sid uint64, done func()) { - dlen := sectorbuilder.UserBytesForSectorSize(sectorSize) - - var err error - r := io.LimitReader(rand.New(rand.NewSource(42+int64(sid))), int64(dlen)) - s.ppi, err = sb.AddPiece(dlen, sid, r, []uint64{}) - if err != nil { - t.Fatalf("%+v", err) - } - - s.ticket = sectorbuilder.SealTicket{ - BlockHeight: 5, - TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2}, - } - - s.pco, err = sb.SealPreCommit(sid, s.ticket, []sectorbuilder.PublicPieceInfo{s.ppi}) - if err != nil { - t.Fatalf("%+v", err) - } - - done() -} - -func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()) { - seed := sectorbuilder.SealSeed{ - BlockHeight: 15, - TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}, - } - - proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco) - if err != nil { - t.Fatalf("%+v", err) - } - - ok, err := sectorbuilder.VerifySeal(sectorSize, s.pco.CommR[:], s.pco.CommD[:], sb.Miner, s.ticket.TicketBytes[:], seed.TicketBytes[:], s.sid, proof) - if err != nil { - t.Fatalf("%+v", err) - } - - if !ok { - t.Fatal("proof failed to validate") - } - - done() -} - -func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time { - cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9} - - ppi := make([]ffi.PublicSectorInfo, len(seals)) - for i, s := range seals { - ppi[i] = ffi.PublicSectorInfo{ - SectorID: s.sid, - CommR: s.pco.CommR, - } - } - - ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi) - - candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{}) - if err != nil { - t.Fatalf("%+v", err) - } - - genCandidates := time.Now() - - if len(candndates) != 1 { - t.Fatal("expected 1 candidate") - } - - postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates) - if err != nil { - t.Fatalf("%+v", err) - } - - ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner) - if err != nil { - t.Fatalf("%+v", err) - } - if !ok { - t.Fatal("bad post") - } - - return genCandidates -} - -func TestSealAndVerify(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode") - } - if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware - t.Skip("this is slow") - } - _ = os.Setenv("RUST_LOG", "info") - - build.SectorSizes = []uint64{sectorSize} - - if err := paramfetch.GetParams(sectorSize); err != nil { - t.Fatalf("%+v", err) - } - - ds := datastore.NewMapDatastore() - - dir, err := ioutil.TempDir("", "sbtest") - if err != nil { - t.Fatal(err) - } - - sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - cleanup := func() { - if t.Failed() { - fmt.Printf("not removing %s\n", dir) - return - } - if err := os.RemoveAll(dir); err != nil { - t.Error(err) - } - } - defer cleanup() - - si, err := sb.AcquireSectorId() - if err != nil { - t.Fatalf("%+v", err) - } - - s := seal{sid: si} - - start := time.Now() - - s.precommit(t, sb, 1, func() {}) - - precommit := time.Now() - - s.commit(t, sb, func() {}) - - commit := time.Now() - - genCandidiates := post(t, sb, s) - - epost := time.Now() - - // Restart sectorbuilder, re-run post - sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - - post(t, sb, s) - - fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) - fmt.Printf("Commit: %s\n", commit.Sub(precommit).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) -} - -func TestSealPoStNoCommit(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode") - } - if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware - t.Skip("this is slow") - } - _ = os.Setenv("RUST_LOG", "info") - - build.SectorSizes = []uint64{sectorSize} - - if err := paramfetch.GetParams(sectorSize); err != nil { - t.Fatalf("%+v", err) - } - - ds := datastore.NewMapDatastore() - - dir, err := ioutil.TempDir("", "sbtest") - if err != nil { - t.Fatal(err) - } - - sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - cleanup := func() { - if t.Failed() { - fmt.Printf("not removing %s\n", dir) - return - } - if err := os.RemoveAll(dir); err != nil { - t.Error(err) - } - } - defer cleanup() - - si, err := sb.AcquireSectorId() - if err != nil { - t.Fatalf("%+v", err) - } - - s := seal{sid: si} - - start := time.Now() - - s.precommit(t, sb, 1, func() {}) - - precommit := time.Now() - - // Restart sectorbuilder, re-run post - sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - - if err := sb.TrimCache(1); err != nil { - t.Fatal(err) - } - - genCandidiates := post(t, sb, s) - - epost := time.Now() - - fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) - fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String()) - fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) -} - -func TestSealAndVerify2(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode") - } - if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware - t.Skip("this is slow") - } - _ = os.Setenv("RUST_LOG", "info") - - build.SectorSizes = []uint64{sectorSize} - - if err := paramfetch.GetParams(sectorSize); err != nil { - t.Fatalf("%+v", err) - } - - ds := datastore.NewMapDatastore() - - dir, err := ioutil.TempDir("", "sbtest") - if err != nil { - t.Fatal(err) - } - - sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - cleanup := func() { - if err := os.RemoveAll(dir); err != nil { - t.Error(err) - } - } - - defer cleanup() - - var wg sync.WaitGroup - - si1, err := sb.AcquireSectorId() - if err != nil { - t.Fatalf("%+v", err) - } - si2, err := sb.AcquireSectorId() - if err != nil { - t.Fatalf("%+v", err) - } - - s1 := seal{sid: si1} - s2 := seal{sid: si2} - - wg.Add(2) - go s1.precommit(t, sb, 1, wg.Done) - time.Sleep(100 * time.Millisecond) - go s2.precommit(t, sb, 2, wg.Done) - wg.Wait() - - wg.Add(2) - go s1.commit(t, sb, wg.Done) - go s2.commit(t, sb, wg.Done) - wg.Wait() - - post(t, sb, s1, s2) -} - -func TestAcquireID(t *testing.T) { - ds := datastore.NewMapDatastore() - - dir, err := ioutil.TempDir("", "sbtest") - if err != nil { - t.Fatal(err) - } - - sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - - assertAcquire := func(expect uint64) { - id, err := sb.AcquireSectorId() - require.NoError(t, err) - assert.Equal(t, expect, id) - } - - assertAcquire(1) - assertAcquire(2) - assertAcquire(3) - - sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds) - if err != nil { - t.Fatalf("%+v", err) - } - - assertAcquire(4) - assertAcquire(5) - assertAcquire(6) - - if err := os.RemoveAll(dir); err != nil { - t.Error(err) - } -} diff --git a/lib/sectorbuilder/simple.go b/lib/sectorbuilder/simple.go deleted file mode 100644 index 28078493a..000000000 --- a/lib/sectorbuilder/simple.go +++ /dev/null @@ -1,75 +0,0 @@ -package sectorbuilder - -import ( - "context" - "io" - - sectorbuilder "github.com/filecoin-project/filecoin-ffi" - "go.opencensus.io/trace" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" -) - -func (sb *SectorBuilder) SectorSize() uint64 { - return sb.ssize -} - -var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector - -func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) { - var commRa, commDa, ticketa, seeda [32]byte - copy(commRa[:], commR) - copy(commDa[:], commD) - copy(ticketa[:], ticket) - copy(seeda[:], seed) - proverIDa := addressToProverID(proverID) - - return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof) -} - -func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo { - return sectorbuilder.NewSortedPrivateSectorInfo(sectors...) -} - -func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo { - return sectorbuilder.NewSortedPublicSectorInfo(sectors...) -} - -func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0) - return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) -} - -func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) { - challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults) - return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) -} - -func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - var challengeSeeda [CommLen]byte - copy(challengeSeeda[:], challengeSeed) - - _, span := trace.StartSpan(ctx, "VerifyPoSt") - defer span.End() - prover := addressToProverID(proverID) - return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover) -} - -func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) { - f, werr, err := toReadableFile(piece, int64(pieceSize)) - if err != nil { - return [32]byte{}, err - } - - commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize) - if err != nil { - return [32]byte{}, err - } - - return commP, werr() -} - -func GenerateDataCommitment(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([CommLen]byte, error) { - return sectorbuilder.GenerateDataCommitment(ssize, pieces) -} diff --git a/lib/statestore/store.go b/lib/statestore/store.go deleted file mode 100644 index 38ce17b39..000000000 --- a/lib/statestore/store.go +++ /dev/null @@ -1,165 +0,0 @@ -package statestore - -import ( - "bytes" - "fmt" - "reflect" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/query" - cbg "github.com/whyrusleeping/cbor-gen" - "go.uber.org/multierr" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-cbor-util" -) - -type StateStore struct { - ds datastore.Datastore -} - -func New(ds datastore.Datastore) *StateStore { - return &StateStore{ds: ds} -} - -func toKey(k interface{}) datastore.Key { - switch t := k.(type) { - case uint64: - return datastore.NewKey(fmt.Sprint(t)) - case fmt.Stringer: - return datastore.NewKey(t.String()) - default: - panic("unexpected key type") - } -} - -func (st *StateStore) Begin(i interface{}, state interface{}) error { - k := toKey(i) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if has { - return xerrors.Errorf("already tracking state for %v", i) - } - - b, err := cborutil.Dump(state) - if err != nil { - return err - } - - return st.ds.Put(k, b) -} - -func (st *StateStore) End(i interface{}) error { - k := toKey(i) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - return st.ds.Delete(k) -} - -func cborMutator(mutator interface{}) func([]byte) ([]byte, error) { - rmut := reflect.ValueOf(mutator) - - return func(in []byte) ([]byte, error) { - state := reflect.New(rmut.Type().In(0).Elem()) - - err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface()) - if err != nil { - return nil, err - } - - out := rmut.Call([]reflect.Value{state}) - - if err := out[0].Interface(); err != nil { - return nil, err.(error) - } - - return cborutil.Dump(state.Interface()) - } -} - -// mutator func(*T) error -func (st *StateStore) Mutate(i interface{}, mutator interface{}) error { - return st.mutate(i, cborMutator(mutator)) -} - -func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error { - k := toKey(i) - has, err := st.ds.Has(k) - if err != nil { - return err - } - if !has { - return xerrors.Errorf("No state for %s", i) - } - - cur, err := st.ds.Get(k) - if err != nil { - return err - } - - mutated, err := mutator(cur) - if err != nil { - return err - } - - return st.ds.Put(k, mutated) -} - -func (st *StateStore) Has(i interface{}) (bool, error) { - return st.ds.Has(toKey(i)) -} - -func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error { - k := toKey(i) - val, err := st.ds.Get(k) - if err != nil { - if xerrors.Is(err, datastore.ErrNotFound) { - return xerrors.Errorf("No state for %s: %w", i, err) - } - return err - } - - return out.UnmarshalCBOR(bytes.NewReader(val)) -} - -// out: *[]T -func (st *StateStore) List(out interface{}) error { - res, err := st.ds.Query(query.Query{}) - if err != nil { - return err - } - defer res.Close() - - outT := reflect.TypeOf(out).Elem().Elem() - rout := reflect.ValueOf(out) - - var errs error - - for { - res, ok := res.NextSync() - if !ok { - break - } - if res.Error != nil { - return res.Error - } - - elem := reflect.New(outT) - err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface()) - if err != nil { - errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err)) - continue - } - - rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem())) - } - - return nil -} diff --git a/lib/statestore/store_test.go b/lib/statestore/store_test.go deleted file mode 100644 index b0aa366d1..000000000 --- a/lib/statestore/store_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package statestore - -import ( - "testing" - - "github.com/ipfs/go-datastore" - - "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/lotus/chain/types" -) - -func TestList(t *testing.T) { - ds := datastore.NewMapDatastore() - - e, err := cborutil.Dump(types.NewInt(7)) - if err != nil { - t.Fatal(err) - } - - if err := ds.Put(datastore.NewKey("/2"), e); err != nil { - t.Fatal(err) - } - - st := &StateStore{ds: ds} - - var out []types.BigInt - if err := st.List(&out); err != nil { - t.Fatal(err) - } - - if len(out) != 1 { - t.Fatal("wrong len") - } - - if out[0].Int64() != 7 { - t.Fatal("wrong data") - } -} diff --git a/node/builder.go b/node/builder.go index f133b2414..ee34b3600 100644 --- a/node/builder.go +++ b/node/builder.go @@ -5,6 +5,7 @@ import ( "errors" "time" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" blockstore "github.com/ipfs/go-ipfs-blockstore" ci "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -30,7 +31,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/hello" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index dd6e6825e..5568ed529 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -13,8 +13,8 @@ import ( files "github.com/ipfs/go-ipfs-files" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/lib/tarutil" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/storage" diff --git a/node/modules/client.go b/node/modules/client.go index 76cb77263..205d39a6d 100644 --- a/node/modules/client.go +++ b/node/modules/client.go @@ -5,7 +5,7 @@ import ( "path/filepath" "reflect" - "github.com/filecoin-project/lotus/lib/statestore" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index dbe3049eb..32b97767d 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -8,8 +8,8 @@ import ( exchange "github.com/ipfs/go-ipfs-exchange-interface" ipld "github.com/ipfs/go-ipld-format" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/lib/statestore" ) // MetadataDS stores metadata diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 5a5b55cec..23fb35455 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -20,13 +20,13 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/deals" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/datatransfer" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/lib/statestore" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" diff --git a/node/node_test.go b/node/node_test.go index e44305977..f8592b889 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -13,6 +13,7 @@ import ( "github.com/libp2p/go-libp2p-core/crypto" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-datastore" badger "github.com/ipfs/go-ds-badger" logging "github.com/ipfs/go-log" @@ -21,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" "github.com/filecoin-project/lotus/api/test" @@ -30,7 +32,6 @@ import ( "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/lib/jsonrpc" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" diff --git a/storage/fpost_run.go b/storage/fpost_run.go index 713a1c072..9e23d1149 100644 --- a/storage/fpost_run.go +++ b/storage/fpost_run.go @@ -5,13 +5,13 @@ import ( "time" ffi "github.com/filecoin-project/filecoin-ffi" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "go.opencensus.io/trace" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (s *fpostScheduler) failPost(eps uint64) { diff --git a/storage/fpost_sched.go b/storage/fpost_sched.go index 4b2a92ac7..fc192ebee 100644 --- a/storage/fpost_sched.go +++ b/storage/fpost_sched.go @@ -4,6 +4,7 @@ import ( "context" "sync" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -11,7 +12,6 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) const Inactive = 0 diff --git a/storage/garbage.go b/storage/garbage.go index 9432d7336..d128413bd 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -7,12 +7,12 @@ import ( "math" "math/rand" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) { diff --git a/storage/miner.go b/storage/miner.go index 2a0b267f5..3265aa2ae 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -13,14 +13,14 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/go-statestore" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" - "github.com/filecoin-project/lotus/lib/statestore" ) var log = logging.Logger("storageminer") diff --git a/storage/sector_states.go b/storage/sector_states.go index ffad0119e..0b86a3059 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -3,13 +3,13 @@ package storage import ( "context" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type providerHandlerFunc func(ctx context.Context, deal SectorInfo) *sectorUpdate diff --git a/storage/sector_types.go b/storage/sector_types.go index 3c328b9c0..2a6f49066 100644 --- a/storage/sector_types.go +++ b/storage/sector_types.go @@ -3,10 +3,10 @@ package storage import ( "context" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error) diff --git a/storage/sector_utils.go b/storage/sector_utils.go index d8d5225bb..664223a20 100644 --- a/storage/sector_utils.go +++ b/storage/sector_utils.go @@ -3,7 +3,7 @@ package storage import ( "math/bits" - "github.com/filecoin-project/lotus/lib/sectorbuilder" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" ) func fillersFromRem(toFill uint64) ([]uint64, error) { diff --git a/storage/sector_utils_test.go b/storage/sector_utils_test.go index 6a975e664..9ab38241c 100644 --- a/storage/sector_utils_test.go +++ b/storage/sector_utils_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/filecoin-project/lotus/lib/sectorbuilder" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" ) func testFill(t *testing.T, n uint64, exp []uint64) { diff --git a/storage/sectorblocks/blocks.go b/storage/sectorblocks/blocks.go index a2ee78053..9d12e0537 100644 --- a/storage/sectorblocks/blocks.go +++ b/storage/sectorblocks/blocks.go @@ -7,6 +7,7 @@ import ( "io" "sync" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" @@ -21,7 +22,6 @@ import ( "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/lib/sectorbuilder" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage" ) diff --git a/storage/sectors.go b/storage/sectors.go index d52dc70b6..e8894749e 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -6,11 +6,11 @@ import ( "io" "math" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" xerrors "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/padreader" - "github.com/filecoin-project/lotus/lib/sectorbuilder" ) const NonceIncrement = math.MaxUint64