cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Cleanup after dropping sectorbuilder

Browse Source
This commit is contained in:
Łukasz Magiera 2020-03-26 20:34:38 +01:00
parent 4be54a976c
commit e050e0307e
27 changed files with 221 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
chain/gen/gen.go
View File

@ -4,8 +4,6 @@ import (
"bytes"
"context"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/minio/blake2b-simd"
"io/ioutil"
"sync/atomic"
@ -23,6 +21,9 @@ import (
format "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
"github.com/ipfs/go-merkledag"
"github.com/minio/blake2b-simd"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
@ -36,8 +37,7 @@ import (
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/node/repo"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
var log = logging.Logger("gen")

19
chain/sync.go
View File

@ -5,20 +5,10 @@ import (
"context"
"errors"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"sync"
"time"
"github.com/minio/blake2b-simd"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/Gurpartap/async"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
@ -27,6 +17,7 @@ import (
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/connmgr"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/minio/blake2b-simd"
cbg "github.com/whyrusleeping/cbor-gen"
"github.com/whyrusleeping/pubsub"
"go.opencensus.io/stats"
@ -34,8 +25,13 @@ import (
"golang.org/x/xerrors"
bls "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
amt "github.com/filecoin-project/go-amt-ipld/v2"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/builtin"
"github.com/filecoin-project/specs-actors/actors/builtin/power"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/util/adt"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
@ -47,6 +43,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sigs"
"github.com/filecoin-project/lotus/metrics"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
var log = logging.Logger("chain")

10
chain/vm/syscalls.go
View File

@ -3,15 +3,17 @@ package vm
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
"golang.org/x/xerrors"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-actors/actors/crypto"
"github.com/filecoin-project/specs-actors/actors/runtime"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
func init() {

4
cmd/lotus-bench/main.go
View File

@ -5,8 +5,6 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
"io/ioutil"
"math/big"
"math/rand"
@ -29,6 +27,8 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
)
var log = logging.Logger("lotus-bench")

4
cmd/lotus-seed/main.go
View File

@ -41,7 +41,7 @@ func main() {
Version: build.UserVersion,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sectorbuilder-dir",
Name: "sector-dir",
Value: "~/.genesis-sectors",
},
},
@ -90,7 +90,7 @@ var preSealCmd = &cli.Command{
},
},
Action: func(c *cli.Context) error {
sdir := c.String("sectorbuilder-dir")
sdir := c.String("sector-dir")
sbroot, err := homedir.Expand(sdir)
if err != nil {
return err

4
cmd/lotus-seed/seed/seed.go
View File

@ -7,8 +7,6 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
"io/ioutil"
"os"
"path/filepath"
@ -26,6 +24,8 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)

2
cmd/lotus-storage-miner/init.go
View File

@ -7,7 +7,6 @@ import (
"encoding/binary"
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io/ioutil"
"os"
"path/filepath"
@ -44,6 +43,7 @@ import (
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sealing"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)

2
cmd/lotus/daemon.go
View File

@ -7,7 +7,6 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io/ioutil"
"os"
"runtime/pprof"
@ -36,6 +35,7 @@ import (
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
const (

1
go.mod
View File

@ -93,7 +93,6 @@ require (
github.com/multiformats/go-multiaddr-net v0.1.3
github.com/multiformats/go-multihash v0.0.13
github.com/opentracing/opentracing-go v1.1.0
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
github.com/whyrusleeping/cbor-gen v0.0.0-20200321164527-9340289d0ca7

6
go.sum
View File

@ -130,8 +130,6 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h
github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200311224656-7d83652bdbed h1:4Wn3XEpFOctHFdtPuKWr6ejbxaC9rivWjSp7qw/sOZ0=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200311224656-7d83652bdbed/go.mod h1:xAd/X905Ncgj8kkHsP2pmQUf6MQT2qJTDcOEfkwCjYc=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200325225948-053034f69825 h1:T+ghJLZsXXn7UYoYFXCmoVlGcVOJo7e5VgTP6EnGlpg=
github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200325225948-053034f69825/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0=
github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
@ -685,7 +683,9 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc=
github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@ -710,8 +710,6 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=

2
markets/retrievaladapter/provider.go
View File

@ -2,7 +2,6 @@ package retrievaladapter
import (
"context"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"github.com/filecoin-project/go-address"
@ -15,6 +14,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
type retrievalProviderNode struct {

4
node/builder.go
View File

@ -3,7 +3,6 @@ package node
import (
"context"
"errors"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"time"
blockstore "github.com/ipfs/go-ipfs-blockstore"
@ -61,6 +60,7 @@ import (
"github.com/filecoin-project/lotus/storage/sealing"
"github.com/filecoin-project/lotus/storage/sectorblocks"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -268,7 +268,7 @@ func Online() Option {
Override(new(stores.SectorIndex), From(new(*stores.Index))),
Override(new(dtypes.MinerID), modules.MinerID),
Override(new(dtypes.MinerAddress), modules.MinerAddress),
Override(new(*ffiwrapper.Config), modules.SectorBuilderConfig),
Override(new(*ffiwrapper.Config), modules.ProofsConfig),
Override(new(stores.LocalStorage), From(new(repo.LockedRepo))),
Override(new(sealing.SectorIDCounter), modules.SectorIDCounter),
Override(new(*sectorstorage.Manager), modules.SectorStorage),

5
node/impl/storminer.go
View File

@ -3,7 +3,6 @@ package impl
import (
"context"
"encoding/json"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"net/http"
"os"
"strconv"
@ -23,14 +22,14 @@ import (
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
type StorageMinerAPI struct {
common.CommonAPI
SectorBuilderConfig *ffiwrapper.Config
//SectorBuilder sectorbuilder.Interface
ProofsConfig *ffiwrapper.Config
SectorBlocks *sectorblocks.SectorBlocks
StorageProvider storagemarket.StorageProvider

4
node/modules/storageminer.go
View File

@ -2,7 +2,6 @@ package modules
import (
"context"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"reflect"
"github.com/ipfs/go-bitswap"
@ -51,6 +50,7 @@ import (
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sealing"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -86,7 +86,7 @@ func MinerID(ma dtypes.MinerAddress) (dtypes.MinerID, error) {
return dtypes.MinerID(id), err
}
func SectorBuilderConfig(maddr dtypes.MinerAddress, fnapi lapi.FullNode) (*ffiwrapper.Config, error) {
func ProofsConfig(maddr dtypes.MinerAddress, fnapi lapi.FullNode) (*ffiwrapper.Config, error) {
ssize, err := fnapi.StateMinerSectorSize(context.TODO(), address.Address(maddr), types.EmptyTSK)
if err != nil {
return nil, err

6
scripts/init-network.sh
View File

@ -14,9 +14,9 @@ staging=$(mktemp -d)
make debug
./lotus-seed --sectorbuilder-dir="${sdt0111}" pre-seal --miner-addr=t0111 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
./lotus-seed --sectorbuilder-dir="${sdt0222}" pre-seal --miner-addr=t0222 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
./lotus-seed --sectorbuilder-dir="${sdt0333}" pre-seal --miner-addr=t0333 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
./lotus-seed --sector-dir="${sdt0111}" pre-seal --miner-addr=t0111 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
./lotus-seed --sector-dir="${sdt0222}" pre-seal --miner-addr=t0222 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
./lotus-seed --sector-dir="${sdt0333}" pre-seal --miner-addr=t0333 --sector-offset=0 --sector-size=${SECTOR_SIZE} --num-sectors=${NUM_SECTORS} &
wait

12
storage/miner.go
View File

@ -122,18 +122,18 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
return nil
}
type SectorBuilderEpp struct {
type StorageEpp struct {
prover storage.Prover
miner abi.ActorID
}
func NewElectionPoStProver(sb storage.Prover, miner dtypes.MinerID) *SectorBuilderEpp {
return &SectorBuilderEpp{sb, abi.ActorID(miner)}
func NewElectionPoStProver(sb storage.Prover, miner dtypes.MinerID) *StorageEpp {
return &StorageEpp{sb, abi.ActorID(miner)}
}
var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil)
var _ gen.ElectionPoStProver = (*StorageEpp)(nil)
func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.SectorInfo, rand abi.PoStRandomness) ([]storage.PoStCandidateWithTicket, error) {
func (epp *StorageEpp) GenerateCandidates(ctx context.Context, ssi []abi.SectorInfo, rand abi.PoStRandomness) ([]storage.PoStCandidateWithTicket, error) {
start := time.Now()
var faults []abi.SectorNumber // TODO
@ -145,7 +145,7 @@ func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.S
return cds, nil
}
func (epp *SectorBuilderEpp) ComputeProof(ctx context.Context, ssi []abi.SectorInfo, rand []byte, winners []storage.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
func (epp *StorageEpp) ComputeProof(ctx context.Context, ssi []abi.SectorInfo, rand []byte, winners []storage.PoStCandidateWithTicket) ([]abi.PoStProof, error) {
if build.InsecurePoStValidation {
log.Warn("Generating fake EPost proof! You should only see this while running tests!")
return []abi.PoStProof{{ProofBytes: []byte("valid proof")}}, nil

2
storage/sealing/checks.go
View File

@ -3,7 +3,6 @@ package sealing
import (
"bytes"
"context"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
"github.com/ipfs/go-cid"
cbg "github.com/whyrusleeping/cbor-gen"
@ -16,6 +15,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
)
// TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting

10
storage/sectorstorage/ffiwrapper/sealer.go
View File

@ -7,7 +7,7 @@ import (
var log = logging.Logger("ffiwrapper")
type SectorBuilder struct {
type Sealer struct {
sealProofType abi.RegisteredProof
postProofType abi.RegisteredProof
ssize abi.SectorSize // a function of sealProofType and postProofType
@ -24,18 +24,18 @@ func fallbackPostChallengeCount(sectors uint64, faults uint64) uint64 {
return challengeCount
}
func (sb *SectorBuilder) Stop() {
func (sb *Sealer) Stop() {
close(sb.stopping)
}
func (sb *SectorBuilder) SectorSize() abi.SectorSize {
func (sb *Sealer) SectorSize() abi.SectorSize {
return sb.ssize
}
func (sb *SectorBuilder) SealProofType() abi.RegisteredProof {
func (sb *Sealer) SealProofType() abi.RegisteredProof {
return sb.sealProofType
}
func (sb *SectorBuilder) PoStProofType() abi.RegisteredProof {
func (sb *Sealer) PoStProofType() abi.RegisteredProof {
return sb.postProofType
}

82
storage/sectorstorage/ffiwrapper/sealer_cgo.go
View File

@ -19,15 +19,15 @@ import (
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
)
var _ Basic = &SectorBuilder{}
var _ Storage = &Sealer{}
func New(sectors SectorProvider, cfg *Config) (*SectorBuilder, error) {
func New(sectors SectorProvider, cfg *Config) (*Sealer, error) {
sectorSize, err := sizeFromConfig(*cfg)
if err != nil {
return nil, err
}
sb := &SectorBuilder{
sb := &Sealer{
sealProofType: cfg.SealProofType,
postProofType: cfg.PoStProofType,
ssize: sectorSize,
@ -40,47 +40,13 @@ func New(sectors SectorProvider, cfg *Config) (*SectorBuilder, error) {
return sb, nil
}
func (sb *SectorBuilder) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, faults []abi.SectorNumber) (ffi.SortedPrivateSectorInfo, error) {
fmap := map[abi.SectorNumber]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []ffi.PrivateSectorInfo
for _, s := range sectorInfo {
if _, faulty := fmap[s.SectorNumber]; faulty {
continue
}
paths, done, err := sb.sectors.AcquireSector(ctx, abi.SectorID{Miner: mid, Number: s.SectorNumber}, stores.FTCache|stores.FTSealed, 0, false)
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquire sector paths: %w", err)
}
done() // TODO: This is a tiny bit suboptimal
postProofType, err := s.RegisteredProof.RegisteredPoStProof()
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquiring registered PoSt proof from sector info %+v: %w", s, err)
}
out = append(out, ffi.PrivateSectorInfo{
CacheDirPath: paths.Cache,
PoStProofType: postProofType,
SealedSectorPath: paths.Sealed,
SectorInfo: s,
})
}
return ffi.NewSortedPrivateSectorInfo(out...), nil
}
func (sb *SectorBuilder) NewSector(ctx context.Context, sector abi.SectorID) error {
func (sb *Sealer) NewSector(ctx context.Context, sector abi.SectorID) error {
// TODO: Allocate the sector here instead of in addpiece
return nil
}
func (sb *SectorBuilder) AddPiece(ctx context.Context, sector abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) {
func (sb *Sealer) AddPiece(ctx context.Context, sector abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) {
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return abi.PieceInfo{}, err
@ -143,7 +109,7 @@ func (sb *SectorBuilder) AddPiece(ctx context.Context, sector abi.SectorID, exis
}, werr()
}
func (sb *SectorBuilder) ReadPieceFromSealedSector(ctx context.Context, sector abi.SectorID, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealedCID cid.Cid) (io.ReadCloser, error) {
func (sb *Sealer) ReadPieceFromSealedSector(ctx context.Context, sector abi.SectorID, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealedCID cid.Cid) (io.ReadCloser, error) {
path, doneUnsealed, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTUnsealed, false)
if err != nil {
return nil, xerrors.Errorf("acquire unsealed sector path: %w", err)
@ -212,7 +178,7 @@ func (sb *SectorBuilder) ReadPieceFromSealedSector(ctx context.Context, sector a
}, nil
}
func (sb *SectorBuilder) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
func (sb *Sealer) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache, true)
if err != nil {
return nil, xerrors.Errorf("acquiring sector paths: %w", err)
@ -269,7 +235,7 @@ func (sb *SectorBuilder) SealPreCommit1(ctx context.Context, sector abi.SectorID
return p1o, nil
}
func (sb *SectorBuilder) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) {
func (sb *Sealer) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true)
if err != nil {
return storage.SectorCids{}, xerrors.Errorf("acquiring sector paths: %w", err)
@ -287,7 +253,7 @@ func (sb *SectorBuilder) SealPreCommit2(ctx context.Context, sector abi.SectorID
}, nil
}
func (sb *SectorBuilder) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
func (sb *Sealer) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true)
if err != nil {
return nil, xerrors.Errorf("acquire sector paths: %w", err)
@ -314,37 +280,11 @@ func (sb *SectorBuilder) SealCommit1(ctx context.Context, sector abi.SectorID, t
return output, nil
}
func (sb *SectorBuilder) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (storage.Proof, error) {
func (sb *Sealer) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (storage.Proof, error) {
return ffi.SealCommitPhase2(phase1Out, sector.Number, sector.Miner)
}
func (sb *SectorBuilder) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return storage.FallbackPostOut{}, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
challengeSeed[31] = 0
candidates, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return storage.FallbackPostOut{}, err
}
winners := make([]abi.PoStCandidate, len(candidates))
for idx := range winners {
winners[idx] = candidates[idx].Candidate
}
proof, err := ffi.GeneratePoSt(miner, privsectors, challengeSeed, winners)
return storage.FallbackPostOut{
PoStInputs: ffiToStorageCandidates(candidates),
Proof: proof,
}, err
}
func (sb *SectorBuilder) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID) error {
paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, false)
if err != nil {
return xerrors.Errorf("acquiring sector cache path: %w", err)

6
storage/sectorstorage/ffiwrapper/sealer_test.go
View File

@ -37,7 +37,7 @@ type seal struct {
ticket abi.SealRandomness
}
func (s *seal) precommit(t *testing.T, sb *SectorBuilder, id abi.SectorID, done func()) {
func (s *seal) precommit(t *testing.T, sb *Sealer, id abi.SectorID, done func()) {
defer done()
dlen := abi.PaddedPieceSize(sectorSize).Unpadded()
@ -61,7 +61,7 @@ func (s *seal) precommit(t *testing.T, sb *SectorBuilder, id abi.SectorID, done
s.cids = cids
}
func (s *seal) commit(t *testing.T, sb *SectorBuilder, done func()) {
func (s *seal) commit(t *testing.T, sb *Sealer, done func()) {
defer done()
seed := abi.InteractiveSealRandomness{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
@ -95,7 +95,7 @@ func (s *seal) commit(t *testing.T, sb *SectorBuilder, done func()) {
}
}
func post(t *testing.T, sb *SectorBuilder, seals ...seal) time.Time {
func post(t *testing.T, sb *Sealer, seals ...seal) time.Time {
randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7}
sis := make([]abi.SectorInfo, len(seals))

9
storage/sectorstorage/ffiwrapper/types.go
View File

@ -3,9 +3,10 @@ package ffiwrapper
import (
"context"
"errors"
"github.com/ipfs/go-cid"
"io"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
@ -20,14 +21,14 @@ type Validator interface {
CanProve(sector stores.SectorPaths) (bool, error)
}
type Sealer interface {
type StorageSealer interface {
storage.Sealer
storage.Storage
}
type Basic interface {
type Storage interface {
storage.Prover
Sealer
StorageSealer
ReadPieceFromSealedSector(context.Context, abi.SectorID, UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error)
}

80
storage/sectorstorage/ffiwrapper/varifier_cgo.go
View File

@ -1,80 +0,0 @@
//+build cgo
package ffiwrapper
import (
"context"
"go.opencensus.io/trace"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
)
func (sb *SectorBuilder) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) {
challengeSeed[31] = 0
privsects, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
return ffi.GeneratePoSt(miner, privsects, challengeSeed, winners)
}
func (sb *SectorBuilder) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return nil, err
}
challengeSeed[31] = 0
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
pc, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, err
}
return ffiToStorageCandidates(pc), nil
}
func ffiToStorageCandidates(pc []ffi.PoStCandidateWithTicket) []storage.PoStCandidateWithTicket {
out := make([]storage.PoStCandidateWithTicket, len(pc))
for i := range out {
out[i] = storage.PoStCandidateWithTicket{
Candidate: pc[i].Candidate,
Ticket: pc[i].Ticket,
}
}
return out
}
var _ Verifier = ProofVerifier
type proofVerifier struct{}
var ProofVerifier = proofVerifier{}
func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) {
return ffi.VerifySeal(info)
}
func (proofVerifier) VerifyElectionPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func (proofVerifier) VerifyFallbackPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func verifyPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
info.Randomness[31] = 0
return ffi.VerifyPoSt(info)
}

142
storage/sectorstorage/ffiwrapper/verifier_cgo.go Normal file
View File

@ -0,0 +1,142 @@
//+build cgo
package ffiwrapper
import (
"context"
"golang.org/x/xerrors"
"go.opencensus.io/trace"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
)
func (sb *Sealer) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) {
challengeSeed[31] = 0
privsects, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, nil) // TODO: faults
if err != nil {
return nil, err
}
return ffi.GeneratePoSt(miner, privsects, challengeSeed, winners)
}
func (sb *Sealer) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return storage.FallbackPostOut{}, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
challengeSeed[31] = 0
candidates, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return storage.FallbackPostOut{}, err
}
winners := make([]abi.PoStCandidate, len(candidates))
for idx := range winners {
winners[idx] = candidates[idx].Candidate
}
proof, err := ffi.GeneratePoSt(miner, privsectors, challengeSeed, winners)
return storage.FallbackPostOut{
PoStInputs: ffiToStorageCandidates(candidates),
Proof: proof,
}, err
}
func (sb *Sealer) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) {
privsectors, err := sb.pubSectorToPriv(ctx, miner, sectorInfo, faults)
if err != nil {
return nil, err
}
challengeSeed[31] = 0
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo)), uint64(len(faults)))
pc, err := ffi.GenerateCandidates(miner, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, err
}
return ffiToStorageCandidates(pc), nil
}
func ffiToStorageCandidates(pc []ffi.PoStCandidateWithTicket) []storage.PoStCandidateWithTicket {
out := make([]storage.PoStCandidateWithTicket, len(pc))
for i := range out {
out[i] = storage.PoStCandidateWithTicket{
Candidate: pc[i].Candidate,
Ticket: pc[i].Ticket,
}
}
return out
}
func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, faults []abi.SectorNumber) (ffi.SortedPrivateSectorInfo, error) {
fmap := map[abi.SectorNumber]struct{}{}
for _, fault := range faults {
fmap[fault] = struct{}{}
}
var out []ffi.PrivateSectorInfo
for _, s := range sectorInfo {
if _, faulty := fmap[s.SectorNumber]; faulty {
continue
}
paths, done, err := sb.sectors.AcquireSector(ctx, abi.SectorID{Miner: mid, Number: s.SectorNumber}, stores.FTCache|stores.FTSealed, 0, false)
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquire sector paths: %w", err)
}
done() // TODO: This is a tiny bit suboptimal
postProofType, err := s.RegisteredProof.RegisteredPoStProof()
if err != nil {
return ffi.SortedPrivateSectorInfo{}, xerrors.Errorf("acquiring registered PoSt proof from sector info %+v: %w", s, err)
}
out = append(out, ffi.PrivateSectorInfo{
CacheDirPath: paths.Cache,
PoStProofType: postProofType,
SealedSectorPath: paths.Sealed,
SectorInfo: s,
})
}
return ffi.NewSortedPrivateSectorInfo(out...), nil
}
var _ Verifier = ProofVerifier
type proofVerifier struct{}
var ProofVerifier = proofVerifier{}
func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) {
return ffi.VerifySeal(info)
}
func (proofVerifier) VerifyElectionPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func (proofVerifier) VerifyFallbackPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
return verifyPost(ctx, info)
}
func verifyPost(ctx context.Context, info abi.PoStVerifyInfo) (bool, error) {
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
info.Randomness[31] = 0
return ffi.VerifyPoSt(info)
}

6
storage/sectorstorage/manager.go
View File

@ -4,7 +4,6 @@ import (
"container/list"
"context"
"errors"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"net/http"
"sync"
@ -19,6 +18,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -30,7 +30,7 @@ var ErrNoWorkers = errors.New("no suitable workers found")
type URLs []string
type Worker interface {
ffiwrapper.Sealer
ffiwrapper.StorageSealer
TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error)
@ -47,7 +47,7 @@ type SectorManager interface {
ReadPieceFromSealedSector(context.Context, abi.SectorID, ffiwrapper.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error)
ffiwrapper.Sealer
ffiwrapper.StorageSealer
storage.Prover
}

8
storage/sectorstorage/mock/mock.go
View File

@ -4,8 +4,6 @@ import (
"bytes"
"context"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"io/ioutil"
"math/big"
@ -20,6 +18,8 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/sectorstorage"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
)
var log = logging.Logger("sbmock")
@ -124,7 +124,7 @@ func (sb *SectorMgr) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticke
ss, ok := sb.sectors[sid]
sb.lk.Unlock()
if !ok {
return nil, xerrors.Errorf("no sector with id %d in sectorbuilder", sid)
return nil, xerrors.Errorf("no sector with id %d in storage", sid)
}
ss.lk.Lock()
@ -237,7 +237,7 @@ func (sb *SectorMgr) FailSector(sid abi.SectorID) error {
defer sb.lk.Unlock()
ss, ok := sb.sectors[sid]
if !ok {
return fmt.Errorf("no such sector in sectorbuilder")
return fmt.Errorf("no such sector in storage")
}
ss.failed = true

4
storage/sectorstorage/worker_local.go
View File

@ -2,7 +2,6 @@ package sectorstorage
import (
"context"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"io"
"os"
@ -14,6 +13,7 @@ import (
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks"
"github.com/filecoin-project/lotus/storage/sectorstorage/stores"
)
@ -87,7 +87,7 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.
}, nil
}
func (l *LocalWorker) sb() (ffiwrapper.Basic, error) {
func (l *LocalWorker) sb() (ffiwrapper.Storage, error) {
return ffiwrapper.New(&localWorkerPathProvider{w: l}, l.scfg)
}

2
storage/sectorstorage/zerocomm/zerocomm_test.go
View File

@ -3,7 +3,6 @@ package zerocomm_test
import (
"bytes"
"fmt"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
"io"
"testing"
@ -12,6 +11,7 @@ import (
"github.com/ipfs/go-cid"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper"
"github.com/filecoin-project/lotus/storage/sectorstorage/zerocomm"
)
func TestComms(t *testing.T) {