switch sectorbuilder from lib to external module

point to correct version of sectorbuilder + ensure go-datastore stays at 0.1.1

replace local address type with go-address

consume paramfetch.GetParams instead of local paramfetch.go

remove constants now defined in sectorbuilder

api/api_storage.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	"github.com/filecoin-project/go-sectorbuilder"
 )
 
 // alias because cbor-gen doesn't like non-alias types

api/apistruct/struct.go

@@ -3,7 +3,7 @@ package apistruct
 import (
 	"context"
 
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/network"

build/genesis.go

@@ -1,6 +1,11 @@
 package build
 
-import rice "github.com/GeertJohan/go.rice"
+import (
+	rice "github.com/GeertJohan/go.rice"
+	logging "github.com/ipfs/go-log"
+)
+
+var log = logging.Logger("build")
 
 func MaybeGenesis() []byte {
 	builtinGen, err := rice.FindBox("genesis")

build/paramfetch.go

@@ -1,189 +0,0 @@
-package build
-
-import (
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-
-	rice "github.com/GeertJohan/go.rice"
-	logging "github.com/ipfs/go-log"
-	"github.com/minio/blake2b-simd"
-	"go.uber.org/multierr"
-	"golang.org/x/xerrors"
-	pb "gopkg.in/cheggaaa/pb.v1"
-)
-
-var log = logging.Logger("build")
-
-//const gateway = "http://198.211.99.118/ipfs/"
-const gateway = "https://ipfs.io/ipfs/"
-const paramdir = "/var/tmp/filecoin-proof-parameters"
-const dirEnv = "FIL_PROOFS_PARAMETER_CACHE"
-
-type paramFile struct {
-	Cid        string `json:"cid"`
-	Digest     string `json:"digest"`
-	SectorSize uint64 `json:"sector_size"`
-}
-
-type fetch struct {
-	wg      sync.WaitGroup
-	fetchLk sync.Mutex
-
-	errs []error
-}
-
-func getParamDir() string {
-	if os.Getenv(dirEnv) == "" {
-		return paramdir
-	}
-	return os.Getenv(dirEnv)
-}
-
-func GetParams(storageSize uint64) error {
-	if err := os.Mkdir(getParamDir(), 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-
-	var params map[string]paramFile
-
-	paramBytes := rice.MustFindBox("proof-params").MustBytes("parameters.json")
-	if err := json.Unmarshal(paramBytes, &params); err != nil {
-		return err
-	}
-
-	ft := &fetch{}
-
-	for name, info := range params {
-		if storageSize != info.SectorSize && strings.HasSuffix(name, ".params") {
-			continue
-		}
-
-		ft.maybeFetchAsync(name, info)
-	}
-
-	return ft.wait()
-}
-
-func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
-	ft.wg.Add(1)
-
-	go func() {
-		defer ft.wg.Done()
-
-		path := filepath.Join(getParamDir(), name)
-
-		err := ft.checkFile(path, info)
-		if !os.IsNotExist(err) && err != nil {
-			log.Warn(err)
-		}
-		if err == nil {
-			return
-		}
-
-		ft.fetchLk.Lock()
-		defer ft.fetchLk.Unlock()
-
-		if err := doFetch(path, info); err != nil {
-			ft.errs = append(ft.errs, xerrors.Errorf("fetching file %s failed: %w", path, err))
-			return
-		}
-		err = ft.checkFile(path, info)
-		if err != nil {
-			ft.errs = append(ft.errs, xerrors.Errorf("checking file %s failed: %w", path, err))
-			err := os.Remove(path)
-			if err != nil {
-				ft.errs = append(ft.errs, xerrors.Errorf("remove file %s failed: %w", path, err))
-			}
-		}
-	}()
-}
-
-func (ft *fetch) checkFile(path string, info paramFile) error {
-	if os.Getenv("TRUST_PARAMS") == "1" {
-		log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
-		return nil
-	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	h := blake2b.New512()
-	if _, err := io.Copy(h, f); err != nil {
-		return err
-	}
-
-	sum := h.Sum(nil)
-	strSum := hex.EncodeToString(sum[:16])
-	if strSum == info.Digest {
-		log.Infof("Parameter file %s is ok", path)
-		return nil
-	}
-
-	return xerrors.Errorf("checksum mismatch in param file %s, %s != %s", path, strSum, info.Digest)
-}
-
-func (ft *fetch) wait() error {
-	ft.wg.Wait()
-	return multierr.Combine(ft.errs...)
-}
-
-func doFetch(out string, info paramFile) error {
-	gw := os.Getenv("IPFS_GATEWAY")
-	if gw == "" {
-		gw = gateway
-	}
-	log.Infof("Fetching %s from %s", out, gw)
-
-	outf, err := os.OpenFile(out, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-	if err != nil {
-		return err
-	}
-	defer outf.Close()
-
-	fStat, err := outf.Stat()
-	if err != nil {
-		return err
-	}
-	header := http.Header{}
-	header.Set("Range", "bytes="+strconv.FormatInt(fStat.Size(), 10)+"-")
-	url, err := url.Parse(gw + info.Cid)
-	if err != nil {
-		return err
-	}
-	log.Infof("GET %s", url)
-
-	req := http.Request{
-		Method: "GET",
-		URL:    url,
-		Header: header,
-		Close:  true,
-	}
-
-	resp, err := http.DefaultClient.Do(&req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	bar := pb.New64(resp.ContentLength)
-	bar.Units = pb.U_BYTES
-	bar.ShowSpeed = true
-	bar.Start()
-
-	_, err = io.Copy(outf, bar.NewProxyReader(resp.Body))
-
-	bar.Finish()
-
-	return err
-}

build/params_shared.go

@@ -56,11 +56,6 @@ const SealRandomnessLookback = Finality
 // Epochs
 const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
 
-// 1 / n
-const SectorChallengeRatioDiv = 25
-
-const MaxFallbackPostChallengeCount = 10
-
 // /////
 // Mining
 

build/proof-params/mkparamfetch.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-function b64() {
-	f="$1"
-	case `uname` in
-	Darwin)
-		base64 -i "$f"
-	;;
-	Linux)
-		base64 "$f" -w 0
-	;;
-	esac
-	printf "unsupported system" 1>&2
-	exit 1
-}
-
-sed "s/{{PARAMSJSON}}/$(b64 build/proof-params/parameters.json)/g" build/proof-params/paramfetch.sh.template > ./build/paramfetch.sh
-chmod +x ./build/paramfetch.sh

build/proof-params/parameters.json

@@ -1,103 +0,0 @@
-{
-  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": {
-    "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9",
-    "digest": "39a9edec3355516674f0d12b926be493",
-    "sector_size": 34359738368
-  },
-  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": {
-    "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX",
-    "digest": "0227ae7df4f2affe529ebafbbc7540ee",
-    "sector_size": 34359738368
-  },
-  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": {
-    "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv",
-    "digest": "991041a354b12c280542741f58c7f2ca",
-    "sector_size": 1024
-  },
-  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": {
-    "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU",
-    "digest": "91fac550e1f9bccab213830bb0c85bd6",
-    "sector_size": 1024
-  },
-  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": {
-    "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4",
-    "digest": "6322eacd2773163ddd51f9ca7d645fc4",
-    "sector_size": 1073741824
-  },
-  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": {
-    "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf",
-    "digest": "0df62745fceac922e3e70847cfc70b52",
-    "sector_size": 1073741824
-  },
-  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": {
-    "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9",
-    "digest": "de9d71e672f286706a1673bd57abdaac",
-    "sector_size": 16777216
-  },
-  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": {
-    "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu",
-    "digest": "7f796d3a0f13499181e44b5eee0cc744",
-    "sector_size": 16777216
-  },
-  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": {
-    "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN",
-    "digest": "151ae0ae183fc141e8c2bebc28e5cc10",
-    "sector_size": 268435456
-  },
-  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": {
-    "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji",
-    "digest": "95eb89588e9d1832aca044c3a13178af",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": {
-    "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX",
-    "digest": "fcd50e2e08a8560a6bb3418e883567ed",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": {
-    "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y",
-    "digest": "872e244d86499fd659082e3bcf3f13e7",
-    "sector_size": 268435456
-  },
-  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": {
-    "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN",
-    "digest": "0a338b941c5f17946340de5fc95cab30",
-    "sector_size": 34359738368
-  },
-  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": {
-    "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF",
-    "digest": "c752e070a6b7aa8b79aa661a6b600b55",
-    "sector_size": 34359738368
-  },
-  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": {
-    "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186",
-    "digest": "efa407fd09202dffd15799a8518e73d3",
-    "sector_size": 1024
-  },
-  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": {
-    "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG",
-    "digest": "b2f09a0ccb62da28c890d5b881c8dcd2",
-    "sector_size": 1024
-  },
-  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": {
-    "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q",
-    "digest": "b53c1916a63839ec345aa2224e9198b7",
-    "sector_size": 1073741824
-  },
-  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": {
-    "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA",
-    "digest": "20d58a3fae7343481f8298a2dd493dd7",
-    "sector_size": 1073741824
-  },
-  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": {
-    "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU",
-    "digest": "7698426202c7e07b26ef056d31485b3a",
-    "sector_size": 16777216
-  },
-  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": {
-    "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax",
-    "digest": "834408e5c3fce6ec5d1bf64e64cee94e",
-    "sector_size": 16777216
-  }
-}
-

build/proof-params/paramfetch.sh.template

@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-PARAMS='{{PARAMSJSON}}'
-
-die() {
-	echo "$@" >&2
-	exit 1
-}
-
-have_binary() {
-	type "$1" > /dev/null 2> /dev/null
-}
-
-check_writable() {
-	printf "" > "$1" && rm "$1"
-}
-
-try_download() {
-	url="$1"
-	output="$2"
-	command="$3"
-	util_name="$(set -- $command; echo "$1")"
-
-	if ! have_binary "$util_name"; then
-		return 1
-	fi
-
-	printf '==> Using %s to download "%s" to "%s"\n' "$util_name" "$url" "$output"
-	if eval "$command"; then
-		echo "==> Download complete!"
-		return
-	else
-		echo "error: couldn't download with $util_name ($?)"
-		return 1
-	fi
-}
-
-download() {
-	dl_url="$1"
-	dl_output="$2"
-
-	test "$#" -eq "2" || die "download requires exactly two arguments, was given $@"
-
-	if ! check_writable "$dl_output"; then
-		die "download error: cannot write to $dl_output"
-	fi
-
-	try_download "$dl_url" "$dl_output" "wget '$dl_url' -O '$dl_output'" && return
-	try_download "$dl_url" "$dl_output" "curl --silent --fail --output '$dl_output' '$dl_url'" && return
-	try_download "$dl_url" "$dl_output" "fetch '$dl_url' -o '$dl_output'" && return
-	try_download "$dl_url" "$dl_output" "http '$dl_url' > '$dl_output'" && return
-	try_download "$dl_url" "$dl_output" "ftp -o '$dl_output' '$dl_url'" && return
-
-	die "Unable to download $dl_url. exiting."
-}
-
-fetch_ipget() {
-  local dest="$1"
-  local cid="$2"
-
-  IPGET_PARAMS="--node=spawn -p=/ip4/138.201.67.219/tcp/4002/ws/ipfs/QmUd6zHcbkbcs7SMxwLs48qZVX3vpcM8errYS7xEczwRMA -p=/ip4/138.201.67.218/tcp/4002/ws/ipfs/QmbVWZQhCGrS7DhgLqWbgvdmKN7JueKCREVanfnVpgyq8x -p=/ip4/94.130.135.167/tcp/4002/ws/ipfs/QmUEMvxS2e7iDrereVYc5SWPauXPyNwxcy9BXZrC1QTcHE -p=/ip4/138.201.68.74/tcp/4001/ipfs/QmdnXwLrC8p1ueiq2Qya8joNvk3TVVDAut7PrikmZwubtR -p=/ip4/138.201.67.220/tcp/4001/ipfs/QmNSYxZAiJHeLdkBg38roksAR9So7Y5eojks1yjEcUtZ7i"
-
-  ./bin/ipget $IPGET_PARAMS -o "$dest" "$cid"
-}
-
-fetch_gateway() {
-  local dest="$1"
-  local cid="$2"
-
-  local url="http://198.211.99.118/ipfs/$cid"
-
-  download "$url" "$dest"
-}
-
-OUT_DIR="/var/tmp/filecoin-proof-parameters"
-
-mkdir -p $OUT_DIR
-printf $PARAMS | base64 -d | jq '. | to_entries | map("'$OUT_DIR'/\(.key) \(.value.cid) \(.value.digest)") | .[]' --raw-output | \
-	while read -r dest cid digest; do
-		if [[ -f "$dest" ]]; then
-			b2=$(b2sum "$dest" | head -c 32)
-			if [[ "$digest" == "$b2" ]]; then
-				echo "$dest exists and has correct hash"
-				continue
-			else
-				echo "$dest has incorrect hash"
-				rm -f "$dest"
-			fi
-		fi
-		echo "downloading $dest"
-
-		fetch_gateway "$dest" "$cid"
-	done

chain/deals/client_utils.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"runtime"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-cid"
 	files "github.com/ipfs/go-ipfs-files"
 	unixfile "github.com/ipfs/go-unixfs/file"
@@ -15,8 +16,7 @@ import (
 	"github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/lotus/datatransfer"
 	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/lib/statestore"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 

chain/gen/gen.go

@@ -11,6 +11,7 @@ import (
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-car"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
@@ -28,7 +29,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/wallet"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/genesis"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/node/repo"
 
 	block "github.com/ipfs/go-block-format"

chain/stmgr/utils.go

@@ -3,13 +3,13 @@ package stmgr
 import (
 	"context"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	ffi "github.com/filecoin-project/filecoin-ffi"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 
 	amt "github.com/filecoin-project/go-amt-ipld"
 	cid "github.com/ipfs/go-cid"

chain/sync.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/Gurpartap/async"
 	bls "github.com/filecoin-project/filecoin-ffi"
 	amt "github.com/filecoin-project/go-amt-ipld"
@@ -35,7 +36,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 var log = logging.Logger("chain")

chain/types/blockheader.go

@@ -3,6 +3,7 @@ package types
 import (
 	"bytes"
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"math/big"
 
 	block "github.com/ipfs/go-block-format"
@@ -214,11 +215,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big
 }
 
 func ElectionPostChallengeCount(sectors uint64, faults int) uint64 {
-	if sectors == 0 {
+	return sectorbuilder.ElectionPostChallengeCount(sectors, faults)
-		return 0
-	}
-	// ceil(sectors / build.SectorChallengeRatioDiv)
-	return (sectors-uint64(faults)-1)/build.SectorChallengeRatioDiv + 1
 }
 
 func (t *Ticket) Equals(ot *Ticket) bool {

cmd/lotus-seal-worker/transfer.go

@@ -6,12 +6,12 @@ import (
 	"net/http"
 	"os"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	files "github.com/ipfs/go-ipfs-files"
 	"golang.org/x/xerrors"
 	"gopkg.in/cheggaaa/pb.v1"
 	"path/filepath"
 
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/lib/tarutil"
 )
 

cmd/lotus-seed/main.go

@@ -7,6 +7,7 @@ import (
 
 	"encoding/json"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	badger "github.com/ipfs/go-ds-badger"
 	logging "github.com/ipfs/go-log"
 	"github.com/mitchellh/go-homedir"
@@ -17,7 +18,6 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/genesis"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 var log = logging.Logger("lotus-seed")

cmd/lotus-seed/seed/seed.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	badger "github.com/ipfs/go-ds-badger"
 	logging "github.com/ipfs/go-log"
 	"golang.org/x/xerrors"
@@ -20,7 +21,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
 	"github.com/filecoin-project/lotus/genesis"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 var log = logging.Logger("preseal")

go.mod

@@ -9,18 +9,18 @@ require (
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/docker/go-units v0.4.0
-	github.com/fatih/color v1.7.0 // indirect
 	github.com/filecoin-project/chain-validation v0.0.3
-	github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed
+	github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590
 	github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f
 	github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878
+	github.com/filecoin-project/go-sectorbuilder v0.0.0-20191220163446-1b44c19a72e9
+	github.com/filecoin-project/go-statestore v0.0.0-20191220165813-c30fad612418
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/google/go-cmp v0.3.1 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.1
 	github.com/hashicorp/go-multierror v1.0.0
@@ -70,11 +70,11 @@ require (
 	github.com/libp2p/go-libp2p-yamux v0.2.1
 	github.com/libp2p/go-maddr-filter v0.0.5
 	github.com/mattn/go-isatty v0.0.9 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mattn/go-sqlite3 v1.12.0
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
 	github.com/minio/sha256-simd v0.1.1
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/mr-tron/base58 v1.1.3 // indirect
 	github.com/multiformats/go-base32 v0.0.3
 	github.com/multiformats/go-multiaddr v0.1.1
 	github.com/multiformats/go-multiaddr-dns v0.2.0
@@ -86,22 +86,19 @@ require (
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/otiai10/copy v1.0.2
 	github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
-	github.com/smartystreets/assertions v1.0.1 // indirect
-	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
 	github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
-	go.opencensus.io v0.22.1
+	go.opencensus.io v0.22.2
 	go.uber.org/dig v1.7.0 // indirect
 	go.uber.org/fx v1.9.0
 	go.uber.org/goleak v0.10.0 // indirect
 	go.uber.org/multierr v1.4.0
 	go.uber.org/zap v1.10.0
 	go4.org v0.0.0-20190313082347-94abd6928b1d // indirect
-	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
+	golang.org/x/net v0.0.0-20190923162816-aa69164e4478 // indirect
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	gopkg.in/cheggaaa/pb.v1 v1.0.28

go.sum

@@ -87,10 +87,18 @@ github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cO
 github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
+github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
+github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
+github.com/filecoin-project/go-sectorbuilder v0.0.0-20191126221632-4c9919a1c4fe h1:bkVofm6oHagxZwzZbh3nDUH8lkhjCRe4S6AevAIxRNg=
+github.com/filecoin-project/go-sectorbuilder v0.0.0-20191126221632-4c9919a1c4fe/go.mod h1:NulUuak/mLUOIX+p5p6E9fCaR9VekiU1wpTgUB+DPgw=
+github.com/filecoin-project/go-sectorbuilder v0.0.0-20191220163446-1b44c19a72e9 h1:6U03ui3B3mSCkZmzXEaM+wAUMy/ozJc9y+BGoCui8e0=
+github.com/filecoin-project/go-sectorbuilder v0.0.0-20191220163446-1b44c19a72e9/go.mod h1:tOSdS//OQy+/03Fe9JDGtfJX/iTQIlYig41ZzKh5cwU=
+github.com/filecoin-project/go-statestore v0.0.0-20191220165813-c30fad612418 h1:6qIy+u0HRpRz7iisUTRxYWhTbAES1kt8R/wn7hCybrU=
+github.com/filecoin-project/go-statestore v0.0.0-20191220165813-c30fad612418/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
@@ -179,6 +187,7 @@ github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds=
 github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@@ -478,6 +487,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -530,6 +540,7 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
@@ -660,6 +671,8 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
+go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY=
@@ -721,6 +734,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

lib/sectorbuilder/files.go

@@ -1,126 +0,0 @@
-package sectorbuilder
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-
-	"golang.org/x/xerrors"
-)
-
-func (sb *SectorBuilder) SectorName(sectorID uint64) string {
-	return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID)
-}
-
-func (sb *SectorBuilder) StagedSectorPath(sectorID uint64) string {
-	return filepath.Join(sb.filesystem.pathFor(dataStaging), sb.SectorName(sectorID))
-}
-
-func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string {
-	return filepath.Join(sb.filesystem.pathFor(dataUnsealed), sb.SectorName(sectorID))
-}
-
-func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
-	return os.OpenFile(sb.StagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
-}
-
-func (sb *SectorBuilder) SealedSectorPath(sectorID uint64) (string, error) {
-	path := filepath.Join(sb.filesystem.pathFor(dataSealed), sb.SectorName(sectorID))
-
-	return path, nil
-}
-
-func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) {
-	dir := filepath.Join(sb.filesystem.pathFor(dataCache), sb.SectorName(sectorID))
-
-	err := os.Mkdir(dir, 0755)
-	if os.IsExist(err) {
-		err = nil
-	}
-
-	return dir, err
-}
-
-func (sb *SectorBuilder) GetPath(typ string, sectorName string) (string, error) {
-	_, found := overheadMul[dataType(typ)]
-	if !found {
-		return "", xerrors.Errorf("unknown sector type: %s", typ)
-	}
-
-	return filepath.Join(sb.filesystem.pathFor(dataType(typ)), sectorName), nil
-}
-
-func (sb *SectorBuilder) TrimCache(sectorID uint64) error {
-	dir, err := sb.sectorCacheDir(sectorID)
-	if err != nil {
-		return xerrors.Errorf("getting cache dir: %w", err)
-	}
-
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return xerrors.Errorf("readdir: %w", err)
-	}
-
-	for _, file := range files {
-		if !strings.HasSuffix(file.Name(), ".dat") { // _aux probably
-			continue
-		}
-		if strings.HasSuffix(file.Name(), "-data-tree-r-last.dat") { // Want to keep
-			continue
-		}
-
-		if err := os.Remove(filepath.Join(dir, file.Name())); err != nil {
-			return xerrors.Errorf("rm %s: %w", file.Name(), err)
-		}
-	}
-
-	return nil
-}
-
-func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
-	f, ok := r.(*os.File)
-	if ok {
-		return f, func() error { return nil }, nil
-	}
-
-	var w *os.File
-
-	f, w, err := os.Pipe()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var wait sync.Mutex
-	var werr error
-
-	wait.Lock()
-	go func() {
-		defer wait.Unlock()
-
-		var copied int64
-		copied, werr = io.CopyN(w, r, n)
-		if werr != nil {
-			log.Warnf("toReadableFile: copy error: %+v", werr)
-		}
-
-		err := w.Close()
-		if werr == nil && err != nil {
-			werr = err
-			log.Warnf("toReadableFile: close error: %+v", err)
-			return
-		}
-		if copied != n {
-			log.Warnf("copied different amount than expected: %d != %d", copied, n)
-			werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
-		}
-	}()
-
-	return f, func() error {
-		wait.Lock()
-		return werr
-	}, nil
-}

lib/sectorbuilder/fs.go

@@ -1,116 +0,0 @@
-package sectorbuilder
-
-import (
-	"github.com/filecoin-project/lotus/chain/types"
-	"golang.org/x/xerrors"
-	"os"
-	"path/filepath"
-	"sync"
-	"syscall"
-)
-
-type dataType string
-
-const (
-	dataCache    dataType = "cache"
-	dataStaging  dataType = "staging"
-	dataSealed   dataType = "sealed"
-	dataUnsealed dataType = "unsealed"
-)
-
-var overheadMul = map[dataType]uint64{ // * sectorSize
-	dataCache:    11, // TODO: check if true for 32G sectors
-	dataStaging:  1,
-	dataSealed:   1,
-	dataUnsealed: 1,
-}
-
-type fs struct {
-	path string
-
-	// in progress actions
-
-	reserved map[dataType]uint64
-
-	lk sync.Mutex
-}
-
-func openFs(dir string) *fs {
-	return &fs{
-		path:     dir,
-		reserved: map[dataType]uint64{},
-	}
-}
-
-func (f *fs) init() error {
-	for _, dir := range []string{f.path,
-		f.pathFor(dataCache),
-		f.pathFor(dataStaging),
-		f.pathFor(dataSealed),
-		f.pathFor(dataUnsealed)} {
-		if err := os.Mkdir(dir, 0755); err != nil {
-			if os.IsExist(err) {
-				continue
-			}
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (f *fs) pathFor(typ dataType) string {
-	_, found := overheadMul[typ]
-	if !found {
-		panic("unknown data path requested")
-	}
-
-	return filepath.Join(f.path, string(typ))
-}
-
-func (f *fs) reservedBytes() int64 {
-	var out int64
-	for _, r := range f.reserved {
-		out += int64(r)
-	}
-	return out
-}
-
-func (f *fs) reserve(typ dataType, size uint64) error {
-	f.lk.Lock()
-	defer f.lk.Unlock()
-
-	var fsstat syscall.Statfs_t
-
-	if err := syscall.Statfs(f.pathFor(typ), &fsstat); err != nil {
-		return err
-	}
-
-	fsavail := int64(fsstat.Bavail) * int64(fsstat.Bsize)
-
-	avail := fsavail - f.reservedBytes()
-
-	need := overheadMul[typ] * size
-
-	if int64(need) > avail {
-		return xerrors.Errorf("not enough space in '%s', need %s, available %s (fs: %s, reserved: %s)",
-			f.path,
-			types.NewInt(need).SizeStr(),
-			types.NewInt(uint64(avail)).SizeStr(),
-			types.NewInt(uint64(fsavail)).SizeStr(),
-			types.NewInt(uint64(f.reservedBytes())).SizeStr())
-	}
-
-	f.reserved[typ] += need
-
-	return nil
-}
-
-func (f *fs) free(typ dataType, sectorSize uint64) {
-	f.lk.Lock()
-	defer f.lk.Unlock()
-
-	f.reserved[typ] -= overheadMul[typ] * sectorSize
-
-	return
-}

lib/sectorbuilder/mock.go

@@ -1,27 +0,0 @@
-package sectorbuilder
-
-import (
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) {
-	addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq")
-	if err != nil {
-		return nil, err
-	}
-
-	sb, err := New(&Config{
-		SectorSize: sectorSize,
-
-		Dir: dir,
-
-		WorkerThreads: 2,
-		Miner:         addr,
-	}, ds)
-	if err != nil {
-		return nil, err
-	}
-
-	return sb, nil
-}

lib/sectorbuilder/remote.go

@@ -1,174 +0,0 @@
-package sectorbuilder
-
-import (
-	"context"
-
-	"golang.org/x/xerrors"
-)
-
-type WorkerTaskType int
-
-const (
-	WorkerPreCommit WorkerTaskType = iota
-	WorkerCommit
-)
-
-type WorkerTask struct {
-	Type   WorkerTaskType
-	TaskID uint64
-
-	SectorID uint64
-
-	// preCommit
-	SealTicket SealTicket
-	Pieces     []PublicPieceInfo
-
-	// commit
-	SealSeed SealSeed
-	Rspco    RawSealPreCommitOutput
-}
-
-type workerCall struct {
-	task WorkerTask
-	ret  chan SealRes
-}
-
-func (sb *SectorBuilder) AddWorker(ctx context.Context, cfg WorkerCfg) (<-chan WorkerTask, error) {
-	sb.remoteLk.Lock()
-	defer sb.remoteLk.Unlock()
-
-	taskCh := make(chan WorkerTask)
-	r := &remote{
-		sealTasks: taskCh,
-		busy:      0,
-	}
-
-	sb.remoteCtr++
-	sb.remotes[sb.remoteCtr] = r
-
-	go sb.remoteWorker(ctx, r, cfg)
-
-	return taskCh, nil
-}
-
-func (sb *SectorBuilder) returnTask(task workerCall) {
-	var ret chan workerCall
-	switch task.task.Type {
-	case WorkerPreCommit:
-		ret = sb.precommitTasks
-	case WorkerCommit:
-		ret = sb.commitTasks
-	default:
-		log.Error("unknown task type", task.task.Type)
-	}
-
-	go func() {
-		select {
-		case ret <- task:
-		case <-sb.stopping:
-			return
-		}
-	}()
-}
-
-func (sb *SectorBuilder) remoteWorker(ctx context.Context, r *remote, cfg WorkerCfg) {
-	defer log.Warn("Remote worker disconnected")
-
-	defer func() {
-		sb.remoteLk.Lock()
-		defer sb.remoteLk.Unlock()
-
-		for i, vr := range sb.remotes {
-			if vr == r {
-				delete(sb.remotes, i)
-				return
-			}
-		}
-	}()
-
-	precommits := sb.precommitTasks
-	if cfg.NoPreCommit {
-		precommits = nil
-	}
-	commits := sb.commitTasks
-	if cfg.NoCommit {
-		commits = nil
-	}
-
-	for {
-		select {
-		case task := <-commits:
-			sb.doTask(ctx, r, task)
-		case task := <-precommits:
-			sb.doTask(ctx, r, task)
-		case <-ctx.Done():
-			return
-		case <-sb.stopping:
-			return
-		}
-
-		r.lk.Lock()
-		r.busy = 0
-		r.lk.Unlock()
-	}
-}
-
-func (sb *SectorBuilder) doTask(ctx context.Context, r *remote, task workerCall) {
-	resCh := make(chan SealRes)
-
-	sb.remoteLk.Lock()
-	sb.remoteResults[task.task.TaskID] = resCh
-	sb.remoteLk.Unlock()
-
-	// send the task
-	select {
-	case r.sealTasks <- task.task:
-	case <-ctx.Done():
-		sb.returnTask(task)
-		return
-	}
-
-	r.lk.Lock()
-	r.busy = task.task.TaskID
-	r.lk.Unlock()
-
-	// wait for the result
-	select {
-	case res := <-resCh:
-
-		// send the result back to the caller
-		select {
-		case task.ret <- res:
-		case <-ctx.Done():
-			return
-		case <-sb.stopping:
-			return
-		}
-
-	case <-ctx.Done():
-		log.Warnf("context expired while waiting for task %d (sector %d): %s", task.task.TaskID, task.task.SectorID, ctx.Err())
-		return
-	case <-sb.stopping:
-		return
-	}
-}
-
-func (sb *SectorBuilder) TaskDone(ctx context.Context, task uint64, res SealRes) error {
-	sb.remoteLk.Lock()
-	rres, ok := sb.remoteResults[task]
-	if ok {
-		delete(sb.remoteResults, task)
-	}
-	sb.remoteLk.Unlock()
-
-	if !ok {
-		return xerrors.Errorf("task %d not found", task)
-	}
-
-	select {
-	case rres <- res:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
-}

lib/sectorbuilder/scrub.go

@@ -1,84 +0,0 @@
-package sectorbuilder
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	sectorbuilder "github.com/filecoin-project/filecoin-ffi"
-	"golang.org/x/xerrors"
-)
-
-type Fault struct {
-	SectorID uint64
-
-	Err error
-}
-
-func (sb *SectorBuilder) Scrub(sectorSet sectorbuilder.SortedPublicSectorInfo) []*Fault {
-	var faults []*Fault
-
-	for _, sector := range sectorSet.Values() {
-		err := sb.checkSector(sector.SectorID)
-		if err != nil {
-			faults = append(faults, &Fault{SectorID: sector.SectorID, Err: err})
-		}
-	}
-
-	return faults
-}
-
-func (sb *SectorBuilder) checkSector(sectorID uint64) error {
-	cache, err := sb.sectorCacheDir(sectorID)
-	if err != nil {
-		return xerrors.Errorf("getting sector cache dir: %w", err)
-	}
-
-	if err := assertFile(filepath.Join(cache, "p_aux"), 96, 96); err != nil {
-		return err
-	}
-	if err := assertFile(filepath.Join(cache, "sc-01-data-tree-r-last.dat"), (2*sb.ssize)-32, (2*sb.ssize)-32); err != nil {
-		return err
-	}
-
-	// TODO: better validate this
-	if err := assertFile(filepath.Join(cache, "t_aux"), 100, 32000); err != nil { // TODO: what should this actually be?
-		return err
-	}
-
-	dent, err := ioutil.ReadDir(cache)
-	if err != nil {
-		return xerrors.Errorf("reading cache dir %s", cache)
-	}
-	if len(dent) != 3 {
-		return xerrors.Errorf("found %d files in %s, expected 3", len(dent), cache)
-	}
-
-	sealed, err := sb.SealedSectorPath(sectorID)
-	if err != nil {
-		return xerrors.Errorf("getting sealed sector path: %w", err)
-	}
-
-	if err := assertFile(filepath.Join(sealed), sb.ssize, sb.ssize); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func assertFile(path string, minSz uint64, maxSz uint64) error {
-	st, err := os.Stat(path)
-	if err != nil {
-		return xerrors.Errorf("stat %s: %w", path, err)
-	}
-
-	if st.IsDir() {
-		return xerrors.Errorf("expected %s to be a regular file", path)
-	}
-
-	if uint64(st.Size()) < minSz || uint64(st.Size()) > maxSz {
-		return xerrors.Errorf("%s wasn't within size bounds, expected %d < f < %d, got %d", minSz, maxSz, st.Size())
-	}
-
-	return nil
-}

lib/sectorbuilder/simple.go

@@ -1,75 +0,0 @@
-package sectorbuilder
-
-import (
-	"context"
-	"io"
-
-	sectorbuilder "github.com/filecoin-project/filecoin-ffi"
-	"go.opencensus.io/trace"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-func (sb *SectorBuilder) SectorSize() uint64 {
-	return sb.ssize
-}
-
-var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
-
-func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
-	var commRa, commDa, ticketa, seeda [32]byte
-	copy(commRa[:], commR)
-	copy(commDa[:], commD)
-	copy(ticketa[:], ticket)
-	copy(seeda[:], seed)
-	proverIDa := addressToProverID(proverID)
-
-	return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
-}
-
-func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo {
-	return sectorbuilder.NewSortedPrivateSectorInfo(sectors...)
-}
-
-func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo {
-	return sectorbuilder.NewSortedPublicSectorInfo(sectors...)
-}
-
-func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
-	challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), 0)
-	return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
-}
-
-func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address, faults int) (bool, error) {
-	challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), faults)
-	return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
-}
-
-func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
-	var challengeSeeda [CommLen]byte
-	copy(challengeSeeda[:], challengeSeed)
-
-	_, span := trace.StartSpan(ctx, "VerifyPoSt")
-	defer span.End()
-	prover := addressToProverID(proverID)
-	return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover)
-}
-
-func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
-	f, werr, err := toReadableFile(piece, int64(pieceSize))
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	return commP, werr()
-}
-
-func GenerateDataCommitment(ssize uint64, pieces []sectorbuilder.PublicPieceInfo) ([CommLen]byte, error) {
-	return sectorbuilder.GenerateDataCommitment(ssize, pieces)
-}

node/builder.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	ci "github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/host"
@@ -30,7 +31,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/hello"

node/modules/client.go

@@ -5,7 +5,7 @@ import (
 	"path/filepath"
 	"reflect"
 
-	"github.com/filecoin-project/lotus/lib/statestore"
+	"github.com/filecoin-project/go-statestore"
 	"github.com/filecoin-project/lotus/node/modules/helpers"
 	"github.com/ipfs/go-bitswap"
 	"github.com/ipfs/go-bitswap/network"

node/modules/dtypes/storage.go

@@ -8,8 +8,8 @@ import (
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	ipld "github.com/ipfs/go-ipld-format"
 
+	"github.com/filecoin-project/go-statestore"
 	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/lib/statestore"
 )
 
 // MetadataDS stores metadata

node/node_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/ipfs/go-datastore"
 	badger "github.com/ipfs/go-ds-badger"
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	logging "github.com/ipfs/go-log"
 	"github.com/libp2p/go-libp2p-core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@@ -30,7 +31,6 @@ import (
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/lib/jsonrpc"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/impl"

storage/fpost_run.go

@@ -5,13 +5,13 @@ import (
 	"time"
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 func (s *fpostScheduler) failPost(eps uint64) {

storage/fpost_sched.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sync"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
@@ -11,7 +12,6 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 const Inactive = 0

storage/garbage.go

@@ -7,12 +7,12 @@ import (
 	"math"
 	"math/rand"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {

storage/sector_states.go

@@ -3,13 +3,13 @@ package storage
 import (
 	"context"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 type providerHandlerFunc func(ctx context.Context, deal SectorInfo) *sectorUpdate

storage/sector_types.go

@@ -3,10 +3,10 @@ package storage
 import (
 	"context"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error)

storage/sector_utils.go

@@ -3,7 +3,7 @@ package storage
 import (
 	"math/bits"
 
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
 
 func fillersFromRem(toFill uint64) ([]uint64, error) {

storage/sector_utils_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
 
 func testFill(t *testing.T, n uint64, exp []uint64) {

storage/sectorblocks/blocks.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"sync"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
@@ -21,7 +22,6 @@ import (
 	"github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/storage"
 )

storage/sectors.go

@@ -6,11 +6,11 @@ import (
 	"io"
 	"math"
 
+	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	xerrors "golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/lib/sectorbuilder"
 )
 
 const NonceIncrement = math.MaxUint64