cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #472 from filecoin-project/feat/interactive-porep

Browse Source 
Interactive porep
This commit is contained in:
Łukasz Magiera 2019-11-09 00:30:47 +01:00 committed by GitHub
commit 1bf87bf859
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
99 changed files with 4549 additions and 2133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.circleci/config.yml
View File

@ -6,6 +6,7 @@ executors:
golang:
docker:
- image: circleci/golang:1.13
resource_class: 2xlarge
commands:
install-deps:
@ -15,6 +16,8 @@ commands:
prepare:
steps:
- checkout
- run: sudo apt-get update
- run: sudo apt-get install ocl-icd-opencl-dev
- run: git submodule sync
- run: git submodule update --init
download-params:
@ -22,14 +25,14 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
- 'v2-lotus-params-'
- 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
- 'v15-lotus-params-'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params
- save_cache:
name: Save parameters cache
key: 'v2-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
key: 'v15-lotus-params-{{ checksum "build/proof-params/parameters.json" }}'
paths:
- /var/tmp/filecoin-proof-parameters/
@ -155,7 +158,7 @@ jobs:
workflows:
version: 2
version: 2.1
ci:
jobs:
- lint-changes:

3
README.md
View File

@ -21,6 +21,9 @@ In order to run lotus, please do the following:
- bzr (some go dependency needs this)
- jq
- pkg-config
- opencl-headers
- opencl-driver
- opencl-icd-loader
2. Clone this repo & `cd` into it

52
api/api_common.go Normal file
View File

@ -0,0 +1,52 @@
package api
import (
"context"
"fmt"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/build"
)
type Common interface {
// Auth
AuthVerify(ctx context.Context, token string) ([]Permission, error)
AuthNew(ctx context.Context, perms []Permission) ([]byte, error)
// network
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
NetPeers(context.Context) ([]peer.AddrInfo, error)
NetConnect(context.Context, peer.AddrInfo) error
NetAddrsListen(context.Context) (peer.AddrInfo, error)
NetDisconnect(context.Context, peer.ID) error
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error)
// Version provides information about API provider
Version(context.Context) (Version, error)
}
// Version provides various build-time information
type Version struct {
Version string
// APIVersion is a binary encoded semver version of the remote implementing
// this api
//
// See APIVersion in build/version.go
APIVersion uint32
// TODO: git commit / os / genesis cid?
// Seconds
BlockDelay uint64
}
func (v Version) String() string {
vM, vm, vp := build.VersionInts(v.APIVersion)
return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp)
}

97
api/api.go → api/api_full.go
View File

@ -2,46 +2,17 @@ package api
import (
"context"
"fmt"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-filestore"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
)
func init() {
cbor.RegisterCborType(SealedRef{})
}
type Common interface {
// Auth
AuthVerify(ctx context.Context, token string) ([]Permission, error)
AuthNew(ctx context.Context, perms []Permission) ([]byte, error)
// network
NetConnectedness(context.Context, peer.ID) (network.Connectedness, error)
NetPeers(context.Context) ([]peer.AddrInfo, error)
NetConnect(context.Context, peer.AddrInfo) error
NetAddrsListen(context.Context) (peer.AddrInfo, error)
NetDisconnect(context.Context, peer.ID) error
// ID returns peerID of libp2p node backing this API
ID(context.Context) (peer.ID, error)
// Version provides information about API provider
Version(context.Context) (Version, error)
}
// FullNode API is a low-level interface to the Filecoin network full node
type FullNode interface {
Common
@ -103,9 +74,10 @@ type FullNode interface {
// ClientImport imports file under the specified path into filestore
ClientImport(ctx context.Context, path string) (cid.Cid, error)
ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, epochPrice types.BigInt, blocksDuration uint64) (*cid.Cid, error)
ClientGetDealInfo(context.Context, cid.Cid) (*DealInfo, error)
ClientListDeals(ctx context.Context) ([]DealInfo, error)
ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error)
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) // TODO: specify serialization mode we want (defaults to unixfs for now)
ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error)
ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error
ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error)
@ -123,8 +95,8 @@ type FullNode interface {
StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error)
StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error)
StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error)
StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)
StateMinerPower(context.Context, address.Address, *types.TipSet) (MinerPower, error)
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error)
@ -137,6 +109,10 @@ type FullNode interface {
StateMarketBalance(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error)
StateMarketParticipants(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error)
StateMarketDeals(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error)
StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error)
MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error
// MarketFreeBalance
PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error)
PaychList(context.Context) ([]address.Address, error)
@ -152,47 +128,6 @@ type FullNode interface {
PaychVoucherSubmit(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error)
}
// StorageMiner is a low-level interface to the Filecoin network storage miner node
type StorageMiner interface {
Common
ActorAddress(context.Context) (address.Address, error)
// Temp api for testing
StoreGarbageData(context.Context) error
// Get the status of a given sector by ID
SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error)
// List all staged sectors
SectorsList(context.Context) ([]uint64, error)
SectorsRefs(context.Context) (map[string][]SealedRef, error)
CommitmentsList(context.Context) ([]SectorCommitment, error)
}
// Version provides various build-time information
type Version struct {
Version string
// APIVersion is a binary encoded semver version of the remote implementing
// this api
//
// See APIVersion in build/version.go
APIVersion uint32
// TODO: git commit / os / genesis cid?
// Seconds
BlockDelay uint64
}
func (v Version) String() string {
vM, vm, vp := build.VersionInts(v.APIVersion)
return fmt.Sprintf("%s+api%d.%d.%d", v.Version, vM, vm, vp)
}
type Import struct {
Status filestore.Status
Key cid.Cid
@ -229,7 +164,7 @@ type Message struct {
Message *types.Message
}
type SectorInfo struct {
type ChainSectorInfo struct {
SectorID uint64
CommD []byte
CommR []byte
@ -277,12 +212,6 @@ type MinerPower struct {
TotalPower types.BigInt
}
type SealedRef struct {
Piece string
Offset uint64
Size uint32
}
type QueryOffer struct {
Err string
@ -332,14 +261,6 @@ type SyncState struct {
Height uint64
}
type SectorCommitment struct {
SectorID uint64
Miner address.Address
CommitMsg cid.Cid
DealIDs []uint64
}
type SyncStateStage int
const (

96
api/api_storage.go Normal file
View File

@ -0,0 +1,96 @@
package api
import (
"context"
"fmt"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
// alias because cbor-gen doesn't like non-alias types
type SectorState = uint64
const (
UndefinedSectorState SectorState = iota
Empty // TODO: Is this useful
Packing // sector not in sealStore, and not on chain
Unsealed // sealing / queued
PreCommitting // on chain pre-commit
PreCommitted // waiting for seed
Committing
Proving
SectorNoUpdate = UndefinedSectorState
)
func SectorStateStr(s SectorState) string {
switch s {
case UndefinedSectorState:
return "UndefinedSectorState"
case Empty:
return "Empty"
case Packing:
return "Packing"
case Unsealed:
return "Unsealed"
case PreCommitting:
return "PreCommitting"
case PreCommitted:
return "PreCommitted"
case Committing:
return "Committing"
case Proving:
return "Proving"
}
return fmt.Sprintf("<Unknown %d>", s)
}
// StorageMiner is a low-level interface to the Filecoin network storage miner node
type StorageMiner interface {
Common
ActorAddress(context.Context) (address.Address, error)
// Temp api for testing
StoreGarbageData(context.Context) error
// Get the status of a given sector by ID
SectorsStatus(context.Context, uint64) (SectorInfo, error)
// List all staged sectors
SectorsList(context.Context) ([]uint64, error)
SectorsRefs(context.Context) (map[string][]SealedRef, error)
WorkerStats(context.Context) (WorkerStats, error)
}
type WorkerStats struct {
Free int
Reserved int // for PoSt
Total int
}
type SectorInfo struct {
SectorID uint64
State SectorState
CommD []byte
CommR []byte
Proof []byte
Deals []uint64
Ticket sectorbuilder.SealTicket
Seed sectorbuilder.SealSeed
}
type SealedRef struct {
Piece string
Offset uint64
Size uint64
}
type SealedRefs struct {
Refs []SealedRef
}

142
api/cbor_gen.go
View File

@ -127,3 +127,145 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
return nil
}
func (t *SealedRef) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.t.Piece (string) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Piece)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Piece)); err != nil {
return err
}
// t.t.Offset (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Offset))); err != nil {
return err
}
// t.t.Size (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
return err
}
return nil
}
func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Piece (string) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Piece = string(sval)
}
// t.t.Offset (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Offset = uint64(extra)
// t.t.Size (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = uint64(extra)
return nil
}
func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
return err
}
// t.t.Refs ([]api.SealedRef) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil {
return err
}
for _, v := range t.Refs {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
return nil
}
func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Refs ([]api.SealedRef) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Refs: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Refs = make([]SealedRef, extra)
}
for i := 0; i < int(extra); i++ {
var v SealedRef
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Refs[i] = v
}
return nil
}

39
api/struct.go
View File

@ -3,7 +3,6 @@ package api
import (
"context"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/network"
"github.com/libp2p/go-libp2p-core/peer"
@ -82,12 +81,13 @@ type FullNodeStruct struct {
ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"`
ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"`
ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"`
ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"`
ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"`
ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"`
StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*SectorInfo, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"`
StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"`
StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"`
@ -104,6 +104,9 @@ type FullNodeStruct struct {
StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"`
StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"`
MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"`
PaychList func(context.Context) ([]address.Address, error) `perm:"read"`
@ -129,12 +132,11 @@ type StorageMinerStruct struct {
StoreGarbageData func(context.Context) error `perm:"write"`
SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"`
SectorsList func(context.Context) ([]uint64, error) `perm:"read"`
SectorsStatus func(context.Context, uint64) (SectorInfo, error) `perm:"read"`
SectorsList func(context.Context) ([]uint64, error) `perm:"read"`
SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"`
SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"`
CommitmentsList func(context.Context) ([]SectorCommitment, error) `perm:"read"`
WorkerStats func(context.Context) (WorkerStats, error) `perm:"read"`
}
}
@ -195,6 +197,9 @@ func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]Qu
func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) {
return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration)
}
func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) {
return c.Internal.ClientGetDealInfo(ctx, deal)
}
func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) {
return c.Internal.ClientListDeals(ctx)
@ -340,11 +345,11 @@ func (c *FullNodeStruct) SyncSubmitBlock(ctx context.Context, blk *types.BlockMs
return c.Internal.SyncSubmitBlock(ctx, blk)
}
func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) {
func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) {
return c.Internal.StateMinerSectors(ctx, addr, ts)
}
func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*SectorInfo, error) {
func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) {
return c.Internal.StateMinerProvingSet(ctx, addr, ts)
}
@ -411,6 +416,14 @@ func (c *FullNodeStruct) StateMarketDeals(ctx context.Context, ts *types.TipSet)
return c.Internal.StateMarketDeals(ctx, ts)
}
func (c *FullNodeStruct) StateMarketStorageDeal(ctx context.Context, dealid uint64, ts *types.TipSet) (*actors.OnChainDeal, error) {
return c.Internal.StateMarketStorageDeal(ctx, dealid, ts)
}
func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error {
return c.Internal.MarketEnsureAvailable(ctx, addr, amt)
}
func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) {
return c.Internal.PaychGet(ctx, from, to, ensureFunds)
}
@ -468,7 +481,7 @@ func (c *StorageMinerStruct) StoreGarbageData(ctx context.Context) error {
}
// Get the status of a given sector by ID
func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) {
func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) {
return c.Internal.SectorsStatus(ctx, sid)
}
@ -481,8 +494,8 @@ func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]Seal
return c.Internal.SectorsRefs(ctx)
}
func (c *StorageMinerStruct) CommitmentsList(ctx context.Context) ([]SectorCommitment, error) {
return c.Internal.CommitmentsList(ctx)
func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (WorkerStats, error) {
return c.Internal.WorkerStats(ctx)
}
var _ Common = &CommonStruct{}

96
api/test/deals.go Normal file
View File

@ -0,0 +1,96 @@
package test
import (
"context"
"fmt"
"io"
"math/rand"
"os"
"testing"
"time"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl"
)
func TestDealFlow(t *testing.T, b APIBuilder) {
os.Setenv("BELLMAN_NO_GPU", "1")
logging.SetAllLoggers(logging.LevelInfo)
ctx := context.Background()
n, sn := b(t, 1, []int{0})
client := n[0].FullNode.(*impl.FullNodeAPI)
miner := sn[0]
addrinfo, err := client.NetAddrsListen(ctx)
if err != nil {
t.Fatal(err)
}
if err := miner.NetConnect(ctx, addrinfo); err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
r := io.LimitReader(rand.New(rand.NewSource(17)), 1000)
fcid, err := client.ClientImportLocal(ctx, r)
if err != nil {
t.Fatal(err)
}
maddr, err := address.NewFromString("t0101")
if err != nil {
t.Fatal(err)
}
fmt.Println("FILE CID: ", fcid)
mine := true
done := make(chan struct{})
go func() {
defer close(done)
for mine {
time.Sleep(time.Second)
fmt.Println("mining a block now")
if err := n[0].MineOne(ctx); err != nil {
t.Fatal(err)
}
}
}()
deal, err := client.ClientStartDeal(ctx, fcid, maddr, types.NewInt(40000000), 100)
if err != nil {
t.Fatal(err)
}
// TODO: this sleep is only necessary because deals don't immediately get logged in the dealstore, we should fix this
time.Sleep(time.Second)
loop:
for {
di, err := client.ClientGetDealInfo(ctx, *deal)
if err != nil {
t.Fatal(err)
}
switch di.State {
case api.DealRejected:
t.Fatal("deal rejected")
case api.DealFailed:
t.Fatal("deal failed")
case api.DealError:
t.Fatal("deal errored")
case api.DealComplete:
fmt.Println("COMPLETE", di)
break loop
}
fmt.Println("Deal state: ", api.DealStates[di.State])
time.Sleep(time.Second / 2)
}
mine = false
fmt.Println("shutting down mining")
<-done
}

11
api/types.go
View File

@ -25,6 +25,17 @@ const (
DealNoUpdate = DealUnknown
)
var DealStates = []string{
"DealUnknown",
"DealRejected",
"DealAccepted",
"DealStaged",
"DealSealing",
"DealFailed",
"DealComplete",
"DealError",
}
// TODO: check if this exists anywhere else
type MultiaddrSlice []ma.Multiaddr

5
build/paramfetch.go
View File

@ -91,6 +91,11 @@ func (ft *fetch) maybeFetchAsync(name string, info paramFile) {
}
func (ft *fetch) checkFile(path string, info paramFile) error {
if os.Getenv("TRUST_PARAMS") == "1" {
log.Warn("Assuming parameter files are ok. DO NOT USE IN PRODUCTION")
return nil
}
f, err := os.Open(path)
if err != nil {
return err

14
build/params.go
View File

@ -55,7 +55,7 @@ const Finality = 500
// Proofs
// Blocks
const ProvingPeriodDuration = 60
const ProvingPeriodDuration = 300
// PoStChallangeTime sets the window in which post computation should happen
// Blocks
@ -71,6 +71,9 @@ const PoStRandomnessLookback = 1
// Blocks
const SealRandomnessLookback = Finality
// Blocks
const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
// /////
// Mining
@ -81,17 +84,20 @@ const PowerCollateralProportion = 5
const PerCapitaCollateralProportion = 1
const CollateralPrecision = 1000
// Blocks
const InteractivePoRepDelay = 10
// /////
// Devnet settings
const TotalFilecoin = 2000000000
const MiningRewardTotal = 1400000000
const TotalFilecoin = 2_000_000_000
const MiningRewardTotal = 1_400_000_000
const InitialRewardStr = "153856861913558700202"
var InitialReward *big.Int
const FilecoinPrecision = 1000000000000000000
const FilecoinPrecision = 1_000_000_000_000_000_000
// six years
// Blocks

64
build/proof-params/parameters.json
View File

@ -1,82 +1,82 @@
{
"v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": {
"v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": {
"cid": "QmT22f1Np1GpW29NXD7Zrv3Ae4poMYhmkDjyscqL8QrJXY",
"digest": "989fd8d989e0f7f1fe21bb010cf1b231",
"sector_size": 16777216
},
"v14-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": {
"v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": {
"cid": "QmVqSdc23to4UwduCCb25223rpSccvtcgPMfRKY1qjucDc",
"digest": "c6d258c37243b8544238a98100e3e399",
"sector_size": 16777216
},
"v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": {
"v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": {
"cid": "QmRTCqgokEGTMfWVaSr7qFXTNotmpd2QBEi8RsvSQKmPLz",
"digest": "ff77a5e270afc6e1c7fbc19e48348fac",
"sector_size": 1073741824
},
"v14-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": {
"v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": {
"cid": "QmRssVAXRN3xp9VdSpTq1pNjkob3QiikoFZiM5hqrmh1VU",
"digest": "b41f35ac26224258e366327716a835a4",
"sector_size": 1073741824
},
"v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
"v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
"cid": "QmYNVRVzjXkuxJfnHTU5vmEcUBQf8dabXZ4m53SzqMkBv5",
"digest": "d156b685e4a1fe3a1f7230b6a39b5ad4",
"sector_size": 1024
},
"v14-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": {
"v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": {
"cid": "QmaCEcsCFVuepMKdC5WURbr5ucEyLMNGxQaB7HqSnr2KGh",
"digest": "06ff067ac78cdab5d7bbc82170882241",
"sector_size": 1024
},
"v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": {
"v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": {
"cid": "QmVuabRvJ797NwLisGKwRURASGxopBBgg4rfNsbZoSYzAc",
"digest": "0e1ceb79a459a60508f480e5b1fed7ac",
"sector_size": 268435456
},
"v14-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": {
"v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": {
"cid": "QmdWENZBAbuUty1vVNn9vmvj1XbJ5UC8qzpcVD35s5AJxG",
"digest": "1b755c74b9d6823c014f6a7ef76249f2",
"sector_size": 268435456
},
"v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": {
"cid": "QmPG38HmDNVFiQJskqKe9sfSjyHfvZRihcfry78rt22FDT",
"digest": "8ea0b47e72250d5d6dab5d4f859e65de",
"v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": {
"cid": "QmZDVpWTw5Eti5pE7N5z1Cmqsw8hPXhUcvG3cQuceK56LH",
"digest": "6aa80306018ea1328f2d6faf8c080734",
"sector_size": 16777216
},
"v14-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": {
"cid": "Qmd3pNM22pgAoRT24tNyEZmeEWK2GtoZznBvzjie2YgqCn",
"digest": "e39f344757c919ae6bbc9b61311c73b2",
"v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": {
"cid": "QmaoXV7iVSJcfZ5qubYy7NBcXDSdnTzxH85d7M4bdDtfGZ",
"digest": "f6832eb736faf2960e920d32e9780b12",
"sector_size": 16777216
},
"v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": {
"cid": "QmQsS6RqWmgdwPnHCwhBJH3WDPcAxhKfbQUs2bwa8D9su8",
"digest": "09879a69abcc51de5c1095f347c84e2b",
"v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": {
"cid": "QmbUW3a3q5DHBb7Ufk8iSbnSCZgbwpe3serqfwKmcTd11w",
"digest": "64024e461b07c869df2463d33dd28035",
"sector_size": 268435456
},
"v14-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": {
"cid": "QmSFBL5rg2TJv8QjLzrbr4c2KV2uDNN13RBVNUgwemJgM1",
"digest": "db0f245f7e9989879d2fa6328bd57d32",
"v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": {
"cid": "Qme3QgBBE7hUgrK7G9ZfJhzkbvViN5HALFpFduYs5K1piv",
"digest": "32496f4dc434b0ed9ef49cb62497a7d1",
"sector_size": 268435456
},
"v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": {
"cid": "QmZ9UVBfviaNFsKyazA4k8GZcM1tHNDpDyrEK9qkxaMJXx",
"digest": "402a7c7c82eaa4af9fba3c7e4402b65b",
"v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": {
"cid": "QmZzgJmb8WXYDKxS22HDgnoBYcZzXDC7s2c2zsV7kouNZ9",
"digest": "cd91f7ccb2ff57a06f3375946dcbdc68",
"sector_size": 1073741824
},
"v14-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": {
"cid": "QmNmDcPVJ1bFFDcNCcRnEoQ6vNDNpKadLPHyEpUoF47gxV",
"digest": "2741c456346a3758e88249d1f4c0d227",
"v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": {
"cid": "QmRUMVzFnENbvyNb6aN2AJ2dnnewr1ESGA1UQLMVZZdsJM",
"digest": "92fc84b76dbe69c731518aebcb82ac82",
"sector_size": 1073741824
},
"v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": {
"cid": "QmNqgCv6UjdKDNMuiwDweZ22TQYMd3gV6nWiA5PjMtNDVu",
"digest": "a9d316d0dbca152e653d41ad8e40058a",
"v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": {
"cid": "QmSixsGkxJXTAuFfWKy5aEXEDJEnpcb1GkdQVF8TCPWoHy",
"digest": "f8339ae93478ded3840d0bc7efa19953",
"sector_size": 1024
},
"v14-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": {
"cid": "QmTGQGThNFEjaMFgP32YLubACrtpkRoeVEfhDaWi5g6w8u",
"digest": "021b3e81e2980a50fd1ac07424d29a8d",
"v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": {
"cid": "QmTMC8hdZ2TkZ9BFuzHzRLM9SuR2PQdUrSAwABeCuHyV2f",
"digest": "f27f08ce1246ee6612c250bb12803ef1",
"sector_size": 1024
}
}

4
build/version.go
View File

@ -1,7 +1,7 @@
package build
// Version is the local build version, set by build system
const Version = "0.0.0"
const Version = "0.7.0"
// APIVersion is a hex semver version of the rpc api exposed
//
@ -12,7 +12,7 @@ const Version = "0.0.0"
// R R H
// |\vv/|
// vv vv
const APIVersion = 0x000001
const APIVersion = 0x000701
const (
MajorMask = 0xff0000

216
chain/actors/actor_miner.go
View File

@ -2,6 +2,7 @@ package actors
import (
"context"
"encoding/binary"
"fmt"
"github.com/filecoin-project/lotus/build"
@ -21,22 +22,27 @@ import (
type StorageMinerActor struct{}
type StorageMinerActorState struct {
// Contains mostly static info about this miner
Info cid.Cid
// Collateral that is waiting to be withdrawn.
DePledgedCollateral types.BigInt
// Time at which the depledged collateral may be withdrawn.
DePledgeTime types.BigInt
// PreCommittedSectors is the set of sectors that have been committed to but not
// yet had their proofs submitted
PreCommittedSectors map[string]*PreCommittedSector
// All sectors this miner has committed.
Sectors cid.Cid
// TODO: Spec says 'StagedCommittedSectors', which one is it?
// Sectors this miner is currently mining. It is only updated
// when a PoSt is submitted (not as each new sector commitment is added).
ProvingSet cid.Cid
// TODO: these:
// SectorTable
// SectorExpirationQueue
// ChallengeStatus
// Contains mostly static info about this miner
Info cid.Cid
// Faulty sectors reported since last SubmitPost,
// up to the current proving period's challenge time.
CurrentFaultSet types.BitField
@ -52,10 +58,6 @@ type StorageMinerActorState struct {
// removal penalization is needed.
NextDoneSet types.BitField
// Deals this miner has been slashed for since the last post submission.
//TODO: unsupported map key type "Cid" (if you want to use struct keys, your atlas needs a transform to string)
//ArbitratedDeals map[cid.Cid]struct{}
// Amount of power this miner has.
Power types.BigInt
@ -88,6 +90,13 @@ type MinerInfo struct {
// Amount of space in each sector committed to the network by this miner.
SectorSize uint64
// SubsectorCount
}
type PreCommittedSector struct {
Info SectorPreCommitInfo
ReceivedEpoch uint64
}
type StorageMinerConstructorParams struct {
@ -97,9 +106,18 @@ type StorageMinerConstructorParams struct {
PeerID peer.ID
}
type SectorPreCommitInfo struct {
SectorNumber uint64
CommR []byte // TODO: Spec says CID
SealEpoch uint64
DealIDs []uint64
}
type maMethods struct {
Constructor uint64
CommitSector uint64
PreCommitSector uint64
ProveCommitSector uint64
SubmitPoSt uint64
SlashStorageFault uint64
GetCurrentProvingSet uint64
@ -114,32 +132,33 @@ type maMethods struct {
ChangeWorker uint64
IsSlashed uint64
IsLate uint64
AddFaults uint64
DeclareFaults uint64
SlashConsensusFault uint64
}
var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}
var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
func (sma StorageMinerActor) Exports() []interface{} {
return []interface{}{
1: sma.StorageMinerConstructor,
2: sma.CommitSector,
3: sma.SubmitPoSt,
//4: sma.SlashStorageFault,
//5: sma.GetCurrentProvingSet,
//6: sma.ArbitrateDeal,
//7: sma.DePledge,
8: sma.GetOwner,
9: sma.GetWorkerAddr,
10: sma.GetPower,
11: sma.GetPeerID,
12: sma.GetSectorSize,
13: sma.UpdatePeerID,
//14: sma.ChangeWorker,
//15: sma.IsSlashed,
//16: sma.IsLate,
17: sma.AddFaults,
18: sma.SlashConsensusFault,
2: sma.PreCommitSector,
3: sma.ProveCommitSector,
4: sma.SubmitPoSt,
//5: sma.SlashStorageFault,
//6: sma.GetCurrentProvingSet,
//7: sma.ArbitrateDeal,
//8: sma.DePledge,
9: sma.GetOwner,
10: sma.GetWorkerAddr,
11: sma.GetPower, // TODO: Remove
12: sma.GetPeerID,
13: sma.GetSectorSize,
14: sma.UpdatePeerID,
//15: sma.ChangeWorker,
//16: sma.IsSlashed,
//17: sma.IsLate,
18: sma.DeclareFaults,
19: sma.SlashConsensusFault,
}
}
@ -199,23 +218,21 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ
return nil, nil
}
type OnChainSealVerifyInfo struct {
CommD []byte // TODO: update proofs code
CommR []byte
Epoch uint64
Proof []byte
DealIDs []uint64
SectorNumber uint64
}
func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *OnChainSealVerifyInfo) ([]byte, ActorError) {
ctx := context.TODO()
func (sma StorageMinerActor) PreCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorPreCommitInfo) ([]byte, ActorError) {
ctx := vmctx.Context()
oldstate, self, err := loadState(vmctx)
if err != nil {
return nil, err
}
if params.SealEpoch >= vmctx.BlockHeight()+build.SealRandomnessLookback {
return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.SealEpoch, vmctx.BlockHeight()+build.SealRandomnessLookback)
}
if vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit {
return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback)
}
mi, err := loadMinerInfo(vmctx, self)
if err != nil {
return nil, err
@ -225,20 +242,6 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
return nil, aerrors.New(1, "not authorized to commit sector for miner")
}
// TODO: this needs to get normalized to either the ID address or the actor address
maddr := vmctx.Message().To
ticket, err := vmctx.GetRandomness(params.Epoch)
if err != nil {
return nil, aerrors.Wrap(err, "failed to get randomness for commitsector")
}
if ok, err := ValidatePoRep(maddr, mi.SectorSize, params, ticket); err != nil {
return nil, err
} else if !ok {
return nil, aerrors.New(2, "bad proof!")
}
// make sure the miner isnt trying to submit a pre-existing sector
unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber)
if err != nil {
@ -257,10 +260,93 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
return nil, aerrors.New(4, "not enough collateral")
}
self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &PreCommittedSector{
Info: *params,
ReceivedEpoch: vmctx.BlockHeight(),
}
nstate, err := vmctx.Storage().Put(self)
if err != nil {
return nil, err
}
if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
return nil, err
}
return nil, nil
}
func uintToStringKey(i uint64) string {
buf := make([]byte, 10)
n := binary.PutUvarint(buf, i)
return string(buf[:n])
}
type SectorProveCommitInfo struct {
Proof []byte
SectorID uint64
DealIDs []uint64
}
func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
ctx := vmctx.Context()
oldstate, self, err := loadState(vmctx)
if err != nil {
return nil, err
}
mi, err := loadMinerInfo(vmctx, self)
if err != nil {
return nil, err
}
us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)]
if !ok {
return nil, aerrors.New(1, "no pre-commitment found for sector")
}
if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() {
return nil, aerrors.New(2, "too early for proof submission")
}
delete(self.PreCommittedSectors, uintToStringKey(params.SectorID))
// TODO: ensure normalization to ID address
maddr := vmctx.Message().To
ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
if err != nil {
return nil, aerrors.Wrap(err, "failed to get ticket randomness")
}
seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay)
if err != nil {
return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment")
}
enc, err := SerializeParams(&ComputeDataCommitmentParams{
DealIDs: params.DealIDs,
SectorSize: mi.SectorSize,
})
if err != nil {
return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams")
}
commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc)
if err != nil {
return nil, aerrors.Wrap(err, "failed to compute data commitment")
}
if ok, err := ValidatePoRep(maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
return nil, err
} else if !ok {
return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%x)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, params.Proof)
}
// Note: There must exist a unique index in the miner's sector set for each
// sector ID. The `faults`, `recovered`, and `done` parameters of the
// SubmitPoSt method express indices into this sector set.
nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber, params.CommR, params.CommD)
nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.Info.CommR, commD)
if err != nil {
return nil, err
}
@ -270,7 +356,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
// Note: As written here, every miners first PoSt will only be over one sector.
// We could set up a 'grace period' for starting mining that would allow miners
// to submit several sectors for their first proving period. Alternatively, we
// could simply make the 'CommitSector' call take multiple sectors at a time.
// could simply make the 'PreCommitSector' call take multiple sectors at a time.
//
// Note: Proving period is a function of sector size; small sectors take less
// time to prove than large sectors do. Sector size is selected when pledging.
@ -491,6 +577,8 @@ func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID u
return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node")
}
// TODO: Spec says to use SealCommitment, and construct commD from deals each time,
// but that would make SubmitPoSt way, way more expensive
if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil {
return cid.Undef, aerrors.HandleExternalError(err, "failed to set commitment in sector set")
}
@ -525,8 +613,8 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID
return true, comms[0], comms[1], nil
}
func ValidatePoRep(maddr address.Address, ssize uint64, params *OnChainSealVerifyInfo, ticket []byte) (bool, ActorError) {
ok, err := sectorbuilder.VerifySeal(ssize, params.CommR, params.CommD, maddr, ticket, params.SectorNumber, params.Proof)
func ValidatePoRep(maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) {
ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
if err != nil {
return false, aerrors.Absorb(err, 25, "verify seal failed")
}
@ -648,11 +736,11 @@ type PaymentVerifyParams struct {
Proof []byte
}
type AddFaultsParams struct {
type DeclareFaultsParams struct {
Faults types.BitField
}
func (sma StorageMinerActor) AddFaults(act *types.Actor, vmctx types.VMContext, params *AddFaultsParams) ([]byte, ActorError) {
func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
oldstate, self, aerr := loadState(vmctx)
if aerr != nil {
return nil, aerr

55
chain/actors/actor_storagemarket.go
View File

@ -13,6 +13,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type StorageMarketActor struct{}
@ -29,9 +30,10 @@ type smaMethods struct {
SlashStorageDealCollateral uint64
GetLastExpirationFromDealIDs uint64
ActivateStorageDeals uint64
ComputeDataCommitment uint64
}
var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}
var SMAMethods = smaMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
func (sma StorageMarketActor) Exports() []interface{} {
return []interface{}{
@ -45,6 +47,7 @@ func (sma StorageMarketActor) Exports() []interface{} {
// 9: sma.SlashStorageDealCollateral,
// 10: sma.GetLastExpirationFromDealIDs,
11: sma.ActivateStorageDeals, // TODO: move under PublishStorageDeals after specs team approves
12: sma.ComputeDataCommitment,
}
}
@ -433,7 +436,6 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types
deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals)
if err != nil {
// TODO: kind of annoying that this can be caused by gas, otherwise could be fatal
return nil, aerrors.HandleExternalError(err, "loading deals amt")
}
@ -499,7 +501,6 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx
deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals)
if err != nil {
// TODO: kind of annoying that this can be caused by gas, otherwise could be fatal
return nil, aerrors.HandleExternalError(err, "loading deals amt")
}
@ -585,6 +586,54 @@ func transferFunds(from, to, amt types.BigInt) (types.BigInt, types.BigInt) {
return types.BigSub(from, amt), types.BigAdd(to, amt)
}
type ComputeDataCommitmentParams struct {
DealIDs []uint64
SectorSize uint64
}
func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx types.VMContext, params *ComputeDataCommitmentParams) ([]byte, ActorError) {
var self StorageMarketState
old := vmctx.Storage().GetHead()
if err := vmctx.Storage().Get(old, &self); err != nil {
return nil, err
}
deals, err := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Deals)
if err != nil {
return nil, aerrors.HandleExternalError(err, "loading deals amt")
}
var pieces []sectorbuilder.PublicPieceInfo
for _, deal := range params.DealIDs {
var dealInfo OnChainDeal
if err := deals.Get(deal, &dealInfo); err != nil {
if _, is := err.(*amt.ErrNotFound); is {
return nil, aerrors.New(3, "deal not found")
}
return nil, aerrors.HandleExternalError(err, "getting deal info failed")
}
if dealInfo.Deal.Proposal.Provider != vmctx.Message().From {
return nil, aerrors.New(4, "referenced deal was not from caller")
}
var commP [32]byte
copy(commP[:], dealInfo.Deal.Proposal.PieceRef)
pieces = append(pieces, sectorbuilder.PublicPieceInfo{
Size: dealInfo.Deal.Proposal.PieceSize,
CommP: commP,
})
}
commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces)
if err != nil {
return nil, aerrors.Absorb(err, 5, "failed to generate data commitment from pieces")
}
return commd[:], nil
}
/*
func (sma StorageMarketActor) HandleCronAction(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {

1
chain/actors/actor_storagepower.go
View File

@ -329,6 +329,7 @@ func powerLookup(ctx context.Context, vmctx types.VMContext, self *StoragePowerS
return types.EmptyInt, aerrors.New(1, "miner not registered with storage power actor")
}
// TODO: Use local amt
ret, err := vmctx.Send(miner, MAMethods.GetPower, types.NewInt(0), nil)
if err != nil {
return types.EmptyInt, aerrors.Wrap(err, "invoke Miner.GetPower")

469
chain/actors/cbor_gen.go
View File

@ -3,6 +3,7 @@ package actors
import (
"fmt"
"io"
"sort"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
@ -197,24 +198,36 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{140}); err != nil {
if _, err := w.Write([]byte{139}); err != nil {
return err
}
// t.t.Info (cid.Cid) (struct)
// t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map)
{
if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.PreCommittedSectors))); err != nil {
return err
}
if err := cbg.WriteCid(w, t.Info); err != nil {
return xerrors.Errorf("failed to write cid field t.Info: %w", err)
}
keys := make([]string, 0, len(t.PreCommittedSectors))
for k := range t.PreCommittedSectors {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := t.PreCommittedSectors[k]
// t.t.DePledgedCollateral (types.BigInt) (struct)
if err := t.DePledgedCollateral.MarshalCBOR(w); err != nil {
return err
}
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil {
return err
}
if _, err := w.Write([]byte(k)); err != nil {
return err
}
// t.t.DePledgeTime (types.BigInt) (struct)
if err := t.DePledgeTime.MarshalCBOR(w); err != nil {
return err
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
}
// t.t.Sectors (cid.Cid) (struct)
@ -229,6 +242,12 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("failed to write cid field t.ProvingSet: %w", err)
}
// t.t.Info (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Info); err != nil {
return xerrors.Errorf("failed to write cid field t.Info: %w", err)
}
// t.t.CurrentFaultSet (types.BitField) (struct)
if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil {
return err
@ -277,39 +296,62 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 12 {
if extra != 11 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Info (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Info: %w", err)
}
t.Info = c
// t.t.PreCommittedSectors (map[string]*actors.PreCommittedSector) (map)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
// t.t.DePledgedCollateral (types.BigInt) (struct)
{
if err := t.DePledgedCollateral.UnmarshalCBOR(br); err != nil {
return err
}
if maj != cbg.MajMap {
return fmt.Errorf("expected a map (major type 5)")
}
if extra > 4096 {
return fmt.Errorf("t.PreCommittedSectors: map too large")
}
// t.t.DePledgeTime (types.BigInt) (struct)
{
t.PreCommittedSectors = make(map[string]*PreCommittedSector, extra)
if err := t.DePledgeTime.UnmarshalCBOR(br); err != nil {
return err
for i, l := 0, int(extra); i < l; i++ {
var k string
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
k = string(sval)
}
var v *PreCommittedSector
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
v = new(PreCommittedSector)
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
}
}
t.PreCommittedSectors[k] = v
}
// t.t.Sectors (cid.Cid) (struct)
@ -334,6 +376,18 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
t.ProvingSet = c
}
// t.t.Info (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Info: %w", err)
}
t.Info = c
}
// t.t.CurrentFaultSet (types.BitField) (struct)
@ -492,20 +546,17 @@ func (t *StorageMinerConstructorParams) UnmarshalCBOR(r io.Reader) error {
return nil
}
func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error {
func (t *SectorPreCommitInfo) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{134}); err != nil {
if _, err := w.Write([]byte{132}); err != nil {
return err
}
// t.t.CommD ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil {
return err
}
if _, err := w.Write(t.CommD); err != nil {
// t.t.SectorNumber (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil {
return err
}
@ -517,16 +568,8 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.Epoch (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Epoch))); err != nil {
return err
}
// t.t.Proof ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil {
return err
}
if _, err := w.Write(t.Proof); err != nil {
// t.t.SealEpoch (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SealEpoch))); err != nil {
return err
}
@ -539,15 +582,10 @@ func (t *OnChainSealVerifyInfo) MarshalCBOR(w io.Writer) error {
return err
}
}
// t.t.SectorNumber (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil {
return err
}
return nil
}
func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
@ -558,27 +596,20 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 6 {
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.CommD ([]uint8) (slice)
// t.t.SectorNumber (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommD: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommD = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommD); err != nil {
return err
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorNumber = uint64(extra)
// t.t.CommR ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
@ -596,7 +627,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
if _, err := io.ReadFull(br, t.CommR); err != nil {
return err
}
// t.t.Epoch (uint64) (uint64)
// t.t.SealEpoch (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
@ -605,24 +636,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Epoch = uint64(extra)
// t.t.Proof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Proof: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Proof = make([]byte, extra)
if _, err := io.ReadFull(br, t.Proof); err != nil {
return err
}
t.SealEpoch = uint64(extra)
// t.t.DealIDs ([]uint64) (slice)
maj, extra, err = cbg.CborReadHeader(br)
@ -653,7 +667,55 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
t.DealIDs[i] = val
}
// t.t.SectorNumber (uint64) (uint64)
return nil
}
func (t *PreCommittedSector) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.t.Info (actors.SectorPreCommitInfo) (struct)
if err := t.Info.MarshalCBOR(w); err != nil {
return err
}
// t.t.ReceivedEpoch (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ReceivedEpoch))); err != nil {
return err
}
return nil
}
func (t *PreCommittedSector) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Info (actors.SectorPreCommitInfo) (struct)
{
if err := t.Info.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.ReceivedEpoch (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
@ -662,7 +724,7 @@ func (t *OnChainSealVerifyInfo) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorNumber = uint64(extra)
t.ReceivedEpoch = uint64(extra)
return nil
}
@ -1849,23 +1911,31 @@ func (t *PaymentChannelActorState) MarshalCBOR(w io.Writer) error {
}
// t.t.LaneStates (map[string]*actors.LaneState) (map)
if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil {
return err
}
for k, v := range t.LaneStates {
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil {
return err
}
if _, err := w.Write([]byte(k)); err != nil {
{
if err := cbg.CborWriteHeader(w, cbg.MajMap, uint64(len(t.LaneStates))); err != nil {
return err
}
if err := v.MarshalCBOR(w); err != nil {
return err
keys := make([]string, 0, len(t.LaneStates))
for k := range t.LaneStates {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := t.LaneStates[k]
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(k)))); err != nil {
return err
}
if _, err := w.Write([]byte(k)); err != nil {
return err
}
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
}
return nil
}
@ -3598,3 +3668,196 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error {
t.ActivationEpoch = uint64(extra)
return nil
}
func (t *ComputeDataCommitmentParams) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.t.DealIDs ([]uint64) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil {
return err
}
for _, v := range t.DealIDs {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil {
return err
}
}
// t.t.SectorSize (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorSize))); err != nil {
return err
}
return nil
}
func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.DealIDs ([]uint64) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.DealIDs: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.DealIDs = make([]uint64, extra)
}
for i := 0; i < int(extra); i++ {
maj, val, err := cbg.CborReadHeader(br)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj)
}
t.DealIDs[i] = val
}
// t.t.SectorSize (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorSize = uint64(extra)
return nil
}
func (t *SectorProveCommitInfo) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.t.Proof ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil {
return err
}
if _, err := w.Write(t.Proof); err != nil {
return err
}
// t.t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
// t.t.DealIDs ([]uint64) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.DealIDs)))); err != nil {
return err
}
for _, v := range t.DealIDs {
if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, v); err != nil {
return err
}
}
return nil
}
func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Proof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Proof: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Proof = make([]byte, extra)
if _, err := io.ReadFull(br, t.Proof); err != nil {
return err
}
// t.t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
// t.t.DealIDs ([]uint64) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.DealIDs: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.DealIDs = make([]uint64, extra)
}
for i := 0; i < int(extra); i++ {
maj, val, err := cbg.CborReadHeader(br)
if err != nil {
return xerrors.Errorf("failed to read uint64 for t.DealIDs slice: %w", err)
}
if maj != cbg.MajUnsignedInt {
return xerrors.Errorf("value read for array t.DealIDs was not a uint, instead got %d", maj)
}
t.DealIDs[i] = val
}
return nil
}

10
chain/blocksync.go
View File

@ -15,7 +15,7 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/node/modules/dtypes"
blocks "github.com/ipfs/go-block-format"
@ -93,7 +93,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) {
defer s.Close()
var req BlockSyncRequest
if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &req); err != nil {
if err := cborutil.ReadCborRPC(bufio.NewReader(s), &req); err != nil {
log.Errorf("failed to read block sync request: %s", err)
return
}
@ -105,7 +105,7 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) {
return
}
if err := cborrpc.WriteCborRPC(s, resp); err != nil {
if err := cborutil.WriteCborRPC(s, resp); err != nil {
log.Error("failed to write back response for handle stream: ", err)
return
}
@ -401,12 +401,12 @@ func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *Bloc
return nil, err
}
if err := cborrpc.WriteCborRPC(s, req); err != nil {
if err := cborutil.WriteCborRPC(s, req); err != nil {
return nil, err
}
var res BlockSyncResponse
if err := cborrpc.ReadCborRPC(bufio.NewReader(s), &res); err != nil {
if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil {
return nil, err
}

165
chain/deals/cbor_gen.go
View File

@ -118,7 +118,7 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
if _, err := w.Write([]byte{130}); err != nil {
return err
}
@ -126,6 +126,13 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error {
if err := t.DealProposal.MarshalCBOR(w); err != nil {
return err
}
// t.t.Piece (cid.Cid) (struct)
if err := cbg.WriteCid(w, t.Piece); err != nil {
return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
}
return nil
}
@ -140,7 +147,7 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -148,9 +155,33 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
{
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.DealProposal = new(actors.StorageDealProposal)
if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
return err
}
}
}
// t.t.Piece (cid.Cid) (struct)
{
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
}
t.Piece = c
}
return nil
@ -161,7 +192,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{134}); err != nil {
if _, err := w.Write([]byte{133}); err != nil {
return err
}
@ -201,18 +232,6 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
}
}
// t.t.CommitMessage (cid.Cid) (struct)
if t.CommitMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.CommitMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err)
}
}
return nil
}
@ -227,7 +246,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 6 {
if extra != 5 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -307,30 +326,6 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
t.PublishMessage = &c
}
}
// t.t.CommitMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err)
}
t.CommitMessage = &c
}
}
return nil
}
@ -409,7 +404,7 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{135}); err != nil {
if _, err := w.Write([]byte{136}); err != nil {
return err
}
@ -444,6 +439,11 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.t.MinerID (peer.ID) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil {
return err
@ -465,7 +465,7 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 7 {
if extra != 8 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -527,6 +527,15 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
return err
}
}
// t.t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.MinerID (peer.ID) (string)
@ -546,7 +555,7 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{132}); err != nil {
if _, err := w.Write([]byte{135}); err != nil {
return err
}
@ -573,6 +582,29 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
if _, err := w.Write([]byte(t.Miner)); err != nil {
return err
}
// t.t.MinerWorker (address.Address) (struct)
if err := t.MinerWorker.MarshalCBOR(w); err != nil {
return err
}
// t.t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.t.PublishMessage (cid.Cid) (struct)
if t.PublishMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.PublishMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err)
}
}
return nil
}
@ -587,7 +619,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
if extra != 7 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -632,6 +664,49 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
t.Miner = peer.ID(sval)
}
// t.t.MinerWorker (address.Address) (struct)
{
if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.t.PublishMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err)
}
t.PublishMessage = &c
}
}
return nil
}

111
chain/deals/client.go
View File

@ -2,7 +2,6 @@ package deals
import (
"context"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
@ -16,11 +15,15 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/retrieval/discovery"
)
@ -32,6 +35,10 @@ type ClientDeal struct {
Proposal actors.StorageDealProposal
State api.DealState
Miner peer.ID
MinerWorker address.Address
DealID uint64
PublishMessage *cid.Cid
s inet.Stream
}
@ -43,9 +50,10 @@ type Client struct {
w *wallet.Wallet
dag dtypes.ClientDAG
discovery *discovery.Local
mpool full.MpoolAPI
events *events.Events
fm *market.FundMgr
deals ClientStateStore
deals *statestore.StateStore
conns map[cid.Cid]inet.Stream
incoming chan *ClientDeal
@ -59,9 +67,10 @@ type clientDealUpdate struct {
newState api.DealState
id cid.Cid
err error
mut func(*ClientDeal)
}
func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, mpool full.MpoolAPI) *Client {
func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, ds dtypes.MetadataDS, dag dtypes.ClientDAG, discovery *discovery.Local, fm *market.FundMgr, chainapi full.ChainAPI) *Client {
c := &Client{
sm: sm,
chain: chain,
@ -69,9 +78,10 @@ func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *
w: w,
dag: dag,
discovery: discovery,
mpool: mpool,
fm: fm,
events: events.NewEvents(context.TODO(), &chainapi),
deals: ClientStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}},
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))),
conns: map[cid.Cid]inet.Stream{},
incoming: make(chan *ClientDeal, 16),
@ -127,10 +137,13 @@ func (c *Client) onIncoming(deal *ClientDeal) {
}
func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
log.Infof("Deal %s updated state to %d", update.id, update.newState)
log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState])
var deal ClientDeal
err := c.deals.MutateClient(update.id, func(d *ClientDeal) error {
err := c.deals.Mutate(update.id, func(d *ClientDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
}
deal = *d
return nil
})
@ -152,7 +165,8 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
case api.DealStaged:
c.handle(ctx, deal, c.staged, api.DealSealing)
case api.DealSealing:
c.handle(ctx, deal, c.sealing, api.DealComplete)
c.handle(ctx, deal, c.sealing, api.DealNoUpdate)
// TODO: DealComplete -> watch for faults, expiration, etc.
}
}
@ -165,80 +179,59 @@ type ClientDealProposal struct {
ProviderAddress address.Address
Client address.Address
MinerWorker address.Address
MinerID peer.ID
}
func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) {
// check market funds
clientMarketBalance, err := c.sm.MarketBalance(ctx, p.Client, nil)
if err != nil {
return cid.Undef, xerrors.Errorf("getting client market balance failed: %w", err)
if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil {
return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err)
}
if clientMarketBalance.Available.LessThan(types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))) {
// TODO: move to a smarter market funds manager
commP, pieceSize, err := c.commP(ctx, p.Data)
smsg, err := c.mpool.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: p.Client,
Value: types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration)),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance,
})
if err != nil {
return cid.Undef, err
}
_, r, err := c.sm.WaitForMessage(ctx, smsg.Cid())
if err != nil {
return cid.Undef, err
}
if r.ExitCode != 0 {
return cid.Undef, xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode)
}
}
dataSize, err := c.dataSize(ctx, p.Data)
proposal := &actors.StorageDealProposal{
PieceRef: p.Data.Bytes(),
PieceSize: uint64(dataSize),
dealProposal := &actors.StorageDealProposal{
PieceRef: commP,
PieceSize: uint64(pieceSize),
PieceSerialization: actors.SerializationUnixFSv0,
Client: p.Client,
Provider: p.ProviderAddress,
ProposalExpiration: p.ProposalExpiration,
Duration: p.Duration,
StoragePricePerEpoch: p.PricePerEpoch,
StorageCollateral: types.NewInt(uint64(dataSize)), // TODO: real calc
StorageCollateral: types.NewInt(uint64(pieceSize)), // TODO: real calc
}
if err := api.SignWith(ctx, c.w.Sign, p.Client, proposal); err != nil {
if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil {
return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err)
}
proposalNd, err := cborrpc.AsIpld(proposal)
proposalNd, err := cborutil.AsIpld(dealProposal)
if err != nil {
return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err)
}
s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
if err != nil {
s.Reset()
return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
}
if err := cborrpc.WriteCborRPC(s, proposal); err != nil {
proposal := &Proposal{
DealProposal: dealProposal,
Piece: p.Data,
}
if err := cborutil.WriteCborRPC(s, proposal); err != nil {
s.Reset()
return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err)
}
deal := &ClientDeal{
ProposalCid: proposalNd.Cid(),
Proposal: *proposal,
Proposal: *dealProposal,
State: api.DealUnknown,
Miner: p.MinerID,
MinerWorker: p.MinerWorker,
s: s,
}
@ -246,7 +239,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro
c.incoming <- deal
return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{
Address: proposal.Provider,
Address: dealProposal.Provider,
ID: deal.Miner,
})
}
@ -260,12 +253,12 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t
req := &AskRequest{
Miner: a,
}
if err := cborrpc.WriteCborRPC(s, req); err != nil {
if err := cborutil.WriteCborRPC(s, req); err != nil {
return nil, xerrors.Errorf("failed to send ask request: %w", err)
}
var out AskResponse
if err := cborrpc.ReadCborRPC(s, &out); err != nil {
if err := cborutil.ReadCborRPC(s, &out); err != nil {
return nil, xerrors.Errorf("failed to read ask response: %w", err)
}
@ -285,7 +278,19 @@ func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*t
}
func (c *Client) List() ([]ClientDeal, error) {
return c.deals.ListClient()
var out []ClientDeal
if err := c.deals.List(&out); err != nil {
return nil, err
}
return out, nil
}
func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
var out ClientDeal
if err := c.deals.Get(d, &out); err != nil {
return nil, err
}
return &out, nil
}
func (c *Client) Stop() {

219
chain/deals/client_states.go
View File

@ -1,142 +1,225 @@
package deals
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborutil"
)
type clientHandlerFunc func(ctx context.Context, deal ClientDeal) error
type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error)
func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) {
go func() {
err := cb(ctx, deal)
mut, err := cb(ctx, deal)
if err != nil {
next = api.DealError
}
if err == nil && next == api.DealNoUpdate {
return
}
select {
case c.updated <- clientDealUpdate{
newState: next,
id: deal.ProposalCid,
err: err,
mut: mut,
}:
case <-c.stop:
}
}()
}
func (c *Client) new(ctx context.Context, deal ClientDeal) error {
func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
resp, err := c.readStorageDealResp(deal)
if err != nil {
return err
return nil, err
}
if err := c.disconnect(deal); err != nil {
return nil, err
}
/* data transfer happens */
if resp.State != api.DealAccepted {
return xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
}
// TODO: spec says it's optional
pubmsg, err := c.chain.GetMessage(*resp.PublishMessage)
return func(info *ClientDeal) {
info.PublishMessage = resp.PublishMessage
}, nil
}
func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
log.Infow("DEAL ACCEPTED!")
pubmsg, err := c.chain.GetMessage(*deal.PublishMessage)
if err != nil {
return xerrors.Errorf("getting deal pubsish message: %w", err)
return nil, xerrors.Errorf("getting deal pubsish message: %w", err)
}
pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
if err != nil {
return xerrors.Errorf("getting miner worker failed: %w", err)
return nil, xerrors.Errorf("getting miner worker failed: %w", err)
}
if pubmsg.From != pw {
return xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
}
if pubmsg.To != actors.StorageMarketAddress {
return xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
}
if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
return xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
}
var params actors.PublishStorageDealsParams
if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
return nil, err
}
dealIdx := -1
for i, storageDeal := range params.Deals {
// TODO: make it less hacky
eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal)
if err != nil {
return nil, err
}
if eq {
dealIdx = i
break
}
}
if dealIdx == -1 {
return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage)
}
// TODO: timeout
_, ret, err := c.sm.WaitForMessage(ctx, *resp.PublishMessage)
_, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage)
if err != nil {
return xerrors.Errorf("waiting for deal publish message: %w", err)
return nil, xerrors.Errorf("waiting for deal publish message: %w", err)
}
if ret.ExitCode != 0 {
return xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
}
// TODO: persist dealId
log.Info("DEAL ACCEPTED!")
var res actors.PublishStorageDealResponse
if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
return nil, err
}
return nil
return func(info *ClientDeal) {
info.DealID = res.DealIDs[dealIdx]
}, nil
}
func (c *Client) accepted(ctx context.Context, deal ClientDeal) error {
/* data transfer happens */
func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
// TODO: Maybe wait for pre-commit
resp, err := c.readStorageDealResp(deal)
if err != nil {
return err
}
if resp.State != api.DealStaged {
return xerrors.Errorf("deal wasn't staged (State=%d)", resp.State)
}
log.Info("DEAL STAGED!")
return nil
return nil, nil
}
func (c *Client) staged(ctx context.Context, deal ClientDeal) error {
/* miner seals our data, hopefully */
func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
if err != nil {
// TODO: This may be fine for some errors
return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
resp, err := c.readStorageDealResp(deal)
if err != nil {
return err
if sd.ActivationEpoch > 0 {
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
}:
case <-c.stop:
}
return true, false, nil
}
return false, true, nil
}
if resp.State != api.DealSealing {
return xerrors.Errorf("deal wasn't sealed (State=%d)", resp.State)
called := func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error) {
defer func() {
if err != nil {
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
err: xerrors.Errorf("handling applied event: %w", err),
}:
case <-c.stop:
}
}
}()
if msg == nil {
log.Error("timed out waiting for deal activation... what now?")
return false, nil
}
var params actors.SectorProveCommitInfo
if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
return false, err
}
var found bool
for _, dealID := range params.DealIDs {
if dealID == deal.DealID {
found = true
break
}
}
if !found {
return true, nil
}
sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
if err != nil {
return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
}
if sd.ActivationEpoch == 0 {
return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height())
}
log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch)
select {
case c.updated <- clientDealUpdate{
newState: api.DealComplete,
id: deal.ProposalCid,
}:
case <-c.stop:
}
return false, nil
}
log.Info("DEAL SEALED!")
// TODO: want?
/*ssize, err := stmgr.GetMinerSectorSize(ctx, c.sm, nil, deal.Proposal.MinerAddress)
if err != nil {
return xerrors.Errorf("failed to get miner sector size: %w", err)
revert := func(ctx context.Context, ts *types.TipSet) error {
log.Warn("deal activation reverted; TODO: actually handle this!")
// TODO: Just go back to DealSealing?
return nil
}
ok, err := sectorbuilder.VerifyPieceInclusionProof(ssize, deal.Proposal.Size, deal.Proposal.CommP, resp.CommD, resp.PieceInclusionProof.ProofElements)
if err != nil {
return xerrors.Errorf("verifying piece inclusion proof in staged deal %s: %w", deal.ProposalCid, err)
if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, deal.Proposal.Provider, actors.MAMethods.ProveCommitSector); err != nil {
return nil, xerrors.Errorf("failed to set up called handler")
}
if !ok {
return xerrors.Errorf("verifying piece inclusion proof in staged deal %s failed", deal.ProposalCid)
}*/
return nil
}
func (c *Client) sealing(ctx context.Context, deal ClientDeal) error {
resp, err := c.readStorageDealResp(deal)
if err != nil {
return err
}
if resp.State != api.DealComplete {
return xerrors.Errorf("deal wasn't complete (State=%d)", resp.State)
}
// TODO: look for the commit message on chain, negotiate better payment vouchers
log.Info("DEAL COMPLETE!!")
return nil
return nil, nil
}

47
chain/deals/client_utils.go
View File

@ -9,7 +9,9 @@ import (
unixfile "github.com/ipfs/go-unixfs/file"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func (c *Client) failDeal(id cid.Cid, cerr error) {
@ -25,29 +27,41 @@ func (c *Client) failDeal(id cid.Cid, cerr error) {
}
// TODO: store in some sort of audit log
log.Errorf("deal %s failed: %s", id, cerr)
log.Errorf("deal %s failed: %+v", id, cerr)
}
func (c *Client) dataSize(ctx context.Context, data cid.Cid) (int64, error) {
func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) {
root, err := c.dag.Get(ctx, data)
if err != nil {
log.Errorf("failed to get file root for deal: %s", err)
return 0, err
return nil, 0, err
}
n, err := unixfile.NewUnixfsFile(ctx, c.dag, root)
if err != nil {
log.Errorf("cannot open unixfs file: %s", err)
return 0, err
return nil, 0, err
}
uf, ok := n.(files.File)
if !ok {
// TODO: we probably got directory, how should we handle this in unixfs mode?
return 0, xerrors.New("unsupported unixfs type")
return nil, 0, xerrors.New("unsupported unixfs type")
}
return uf.Size()
s, err := uf.Size()
if err != nil {
return nil, 0, err
}
pr, psize := padreader.New(uf, uint64(s))
commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize)
if err != nil {
return nil, 0, xerrors.Errorf("generating CommP: %w", err)
}
return commp[:], psize, nil
}
func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
@ -58,16 +72,29 @@ func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
}
var resp SignedResponse
if err := cborrpc.ReadCborRPC(s, &resp); err != nil {
if err := cborutil.ReadCborRPC(s, &resp); err != nil {
log.Errorw("failed to read Response message", "error", err)
return nil, err
}
// TODO: verify signature
if err := resp.Verify(deal.MinerWorker); err != nil {
return nil, xerrors.Errorf("verifying response signature failed", err)
}
if resp.Response.Proposal != deal.ProposalCid {
return nil, xerrors.New("miner responded to a wrong proposal")
return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid)
}
return &resp.Response, nil
}
func (c *Client) disconnect(deal ClientDeal) error {
s, ok := c.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(c.conns, deal.ProposalCid)
return err
}

48
chain/deals/provider.go
View File

@ -15,9 +15,10 @@ import (
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage/commitment"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -42,15 +43,15 @@ type Provider struct {
ask *types.SignedStorageAsk
askLk sync.Mutex
secst *sectorblocks.SectorBlocks
commt *commitment.Tracker
full api.FullNode
secb *sectorblocks.SectorBlocks
sminer *storage.Miner
full api.FullNode
// TODO: Use a custom protocol or graphsync in the future
// TODO: GC
dag dtypes.StagingDAG
deals MinerStateStore
deals *statestore.StateStore
ds dtypes.MetadataDS
conns map[cid.Cid]inet.Stream
@ -70,7 +71,7 @@ type minerDealUpdate struct {
mut func(*MinerDeal)
}
func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *commitment.Tracker, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) {
func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, fullNode api.FullNode) (*Provider, error) {
addr, err := ds.Get(datastore.NewKey("miner-address"))
if err != nil {
return nil, err
@ -81,13 +82,13 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *
}
h := &Provider{
secst: secst,
commt: commt,
dag: dag,
full: fullNode,
sminer: sminer,
dag: dag,
full: fullNode,
secb: secb,
pricePerByteBlock: types.NewInt(3), // TODO: allow setting
minPieceSize: 1,
minPieceSize: 256, // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up))
conns: map[cid.Cid]inet.Stream{},
@ -98,7 +99,7 @@ func NewProvider(ds dtypes.MetadataDS, secst *sectorblocks.SectorBlocks, commt *
actor: minerAddress,
deals: MinerStateStore{StateStore{ds: namespace.Wrap(ds, datastore.NewKey("/deals/client"))}},
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))),
ds: ds,
}
@ -159,14 +160,14 @@ func (p *Provider) onIncoming(deal MinerDeal) {
}
func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
log.Infof("Deal %s updated state to %d", update.id, update.newState)
log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState])
if update.err != nil {
log.Errorf("deal %s (newSt: %d) failed: %s", update.id, update.newState, update.err)
log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err)
p.failDeal(update.id, update.err)
return
}
var deal MinerDeal
err := p.deals.MutateMiner(update.id, func(d *MinerDeal) error {
err := p.deals.Mutate(update.id, func(d *MinerDeal) error {
d.State = update.newState
if update.mut != nil {
update.mut(d)
@ -191,24 +192,19 @@ func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
}
}
func (p *Provider) newDeal(s inet.Stream, proposal actors.StorageDealProposal) (MinerDeal, error) {
proposalNd, err := cborrpc.AsIpld(&proposal)
if err != nil {
return MinerDeal{}, err
}
ref, err := cid.Cast(proposal.PieceRef)
func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
proposalNd, err := cborutil.AsIpld(proposal.DealProposal)
if err != nil {
return MinerDeal{}, err
}
return MinerDeal{
Client: s.Conn().RemotePeer(),
Proposal: proposal,
Proposal: *proposal.DealProposal,
ProposalCid: proposalNd.Cid(),
State: api.DealUnknown,
Ref: ref,
Ref: proposal.Piece,
s: s,
}, nil
@ -226,7 +222,7 @@ func (p *Provider) HandleStream(s inet.Stream) {
deal, err := p.newDeal(s, proposal)
if err != nil {
log.Error(err)
log.Errorf("%+v", err)
s.Close()
return
}

14
chain/deals/provider_asks.go
View File

@ -8,7 +8,7 @@ import (
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
datastore "github.com/ipfs/go-datastore"
inet "github.com/libp2p/go-libp2p-core/network"
"golang.org/x/xerrors"
@ -54,14 +54,14 @@ func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk {
func (p *Provider) HandleAskStream(s inet.Stream) {
defer s.Close()
var ar AskRequest
if err := cborrpc.ReadCborRPC(s, &ar); err != nil {
if err := cborutil.ReadCborRPC(s, &ar); err != nil {
log.Errorf("failed to read AskRequest from incoming stream: %s", err)
return
}
resp := p.processAskRequest(&ar)
if err := cborrpc.WriteCborRPC(s, resp); err != nil {
if err := cborutil.WriteCborRPC(s, resp); err != nil {
log.Errorf("failed to write ask response: %s", err)
return
}
@ -98,7 +98,7 @@ func (p *Provider) loadAsk() error {
}
var ssa types.SignedStorageAsk
if err := cborrpc.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
return err
}
@ -107,7 +107,7 @@ func (p *Provider) loadAsk() error {
}
func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) {
b, err := cborrpc.Dump(a)
b, err := cborutil.Dump(a)
if err != nil {
return nil, err
}
@ -129,7 +129,7 @@ func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error)
}
func (p *Provider) saveAsk(a *types.SignedStorageAsk) error {
b, err := cborrpc.Dump(a)
b, err := cborutil.Dump(a)
if err != nil {
return err
}
@ -150,7 +150,7 @@ func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error {
return xerrors.Errorf("failed to get worker for miner in ask", err)
}
sigb, err := cborrpc.Dump(ask.Ask)
sigb, err := cborutil.Dump(ask.Ask)
if err != nil {
return xerrors.Errorf("failed to re-serialize ask")
}

133
chain/deals/provider_states.go
View File

@ -4,17 +4,14 @@ import (
"bytes"
"context"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-merkledag"
unixfile "github.com/ipfs/go-unixfs/file"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -41,33 +38,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle
}
// ACCEPTED
func (p *Provider) addMarketFunds(ctx context.Context, worker address.Address, deal MinerDeal) error {
log.Info("Adding market funds for storage collateral")
smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: worker,
Value: deal.Proposal.StorageCollateral,
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance,
})
if err != nil {
return err
}
r, err := p.full.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return err
}
if r.Receipt.ExitCode != 0 {
return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)
}
return nil
}
func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
switch deal.Proposal.PieceSerialization {
//case SerializationRaw:
@ -87,7 +57,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
// TODO: check StorageCollateral
// TODO:
minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30))
if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) {
return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice)
@ -114,16 +83,9 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
return nil, err
}
providerMarketBalance, err := p.full.StateMarketBalance(ctx, waddr, nil)
if err != nil {
return nil, xerrors.Errorf("getting provider market balance failed: %w", err)
}
// TODO: this needs to be atomic
if providerMarketBalance.Available.LessThan(deal.Proposal.StorageCollateral) {
if err := p.addMarketFunds(ctx, waddr, deal); err != nil {
return nil, err
}
// TODO: check StorageCollateral (may be too large (or too small))
if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil {
return nil, err
}
log.Info("publishing deal")
@ -173,15 +135,20 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
log.Info("fetching data for a deal")
mcid := smsg.Cid()
err = p.sendSignedResponse(&Response{
State: api.DealAccepted,
Message: "",
State: api.DealAccepted,
Proposal: deal.ProposalCid,
PublishMessage: &mcid,
StorageDeal: &storageDeal,
})
if err != nil {
return nil, err
}
if err := p.disconnect(deal); err != nil {
log.Warnf("closing client connection: %+v", err)
}
return func(deal *MinerDeal) {
deal.DealID = resp.DealIDs[0]
}, merkledag.FetchGraph(ctx, deal.Ref, p.dag)
@ -190,14 +157,6 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
// STAGED
func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
err := p.sendSignedResponse(&Response{
State: api.DealStaged,
Proposal: deal.ProposalCid,
})
if err != nil {
log.Warnf("Sending deal response failed: %s", err)
}
root, err := p.dag.Get(ctx, deal.Ref)
if err != nil {
return nil, xerrors.Errorf("failed to get file root for deal: %s", err)
@ -221,21 +180,16 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
if err != nil {
return nil, xerrors.Errorf("getting unixfs file size: %w", err)
}
if uint64(size) != deal.Proposal.PieceSize {
return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match unixfs file size")
if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
}
pcid, err := cid.Cast(deal.Proposal.PieceRef)
if err != nil {
return nil, err
}
sectorID, err := p.secst.AddUnixfsPiece(pcid, uf, deal.DealID)
sectorID, err := p.secb.AddUnixfsPiece(ctx, deal.Ref, uf, deal.DealID)
if err != nil {
return nil, xerrors.Errorf("AddPiece failed: %s", err)
}
log.Warnf("New Sector: %d", sectorID)
return func(deal *MinerDeal) {
deal.SectorID = sectorID
}, nil
@ -243,67 +197,14 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
// SEALING
func (p *Provider) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder.SectorSealingStatus, error) {
status, err := p.secst.WaitSeal(ctx, deal.SectorID)
if err != nil {
return sectorbuilder.SectorSealingStatus{}, err
}
switch status.State {
case sealing_state.Sealed:
case sealing_state.Failed:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg)
case sealing_state.Pending:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID)
case sealing_state.Sealing:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID)
default:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID)
}
return status, nil
}
func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
err := p.sendSignedResponse(&Response{
State: api.DealSealing,
Proposal: deal.ProposalCid,
})
if err != nil {
log.Warnf("Sending deal response failed: %s", err)
}
if err := p.secst.SealSector(ctx, deal.SectorID); err != nil {
return nil, xerrors.Errorf("sealing sector failed: %w", err)
}
_, err = p.waitSealed(ctx, deal)
if err != nil {
return nil, err
}
// TODO: Spec doesn't say anything about inclusion proofs anywhere
// Not sure what mechanisms prevents miner from storing data that isn't
// clients' data
// TODO: consider waiting for seal to happen
return nil, nil
}
func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
// TODO: Add dealID to commtracker (probably before sealing)
mcid, err := p.commt.WaitCommit(ctx, deal.Proposal.Provider, deal.SectorID)
if err != nil {
log.Warnf("Waiting for sector commitment message: %s", err)
}
err = p.sendSignedResponse(&Response{
State: api.DealComplete,
Proposal: deal.ProposalCid,
CommitMessage: &mcid,
})
if err != nil {
log.Warnf("Sending deal response failed: %s", err)
}
// TODO: observe sector lifecycle, status, expiration..
return nil, nil
}

33
chain/deals/provider_utils.go
View File

@ -5,11 +5,10 @@ import (
"runtime"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/ipfs/go-cid"
inet "github.com/libp2p/go-libp2p-core/network"
@ -26,7 +25,7 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) {
cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
}
log.Errorf("deal %s failed: %s", id, cerr)
log.Warnf("deal %s failed: %s", id, cerr)
err := p.sendSignedResponse(&Response{
State: api.DealFailed,
@ -45,21 +44,18 @@ func (p *Provider) failDeal(id cid.Cid, cerr error) {
}
}
func (p *Provider) readProposal(s inet.Stream) (proposal actors.StorageDealProposal, err error) {
if err := cborrpc.ReadCborRPC(s, &proposal); err != nil {
func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) {
if err := cborutil.ReadCborRPC(s, &proposal); err != nil {
log.Errorw("failed to read proposal message", "error", err)
return proposal, err
}
if err := proposal.Verify(); err != nil {
if err := proposal.DealProposal.Verify(); err != nil {
return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err)
}
// TODO: Validate proposal maybe
// (and signature, obviously)
if proposal.Provider != p.actor {
log.Errorf("proposal with wrong ProviderAddress: %s", proposal.Provider)
if proposal.DealProposal.Provider != p.actor {
log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider)
return proposal, err
}
@ -72,7 +68,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error {
return xerrors.New("couldn't send response: not connected")
}
msg, err := cborrpc.Dump(resp)
msg, err := cborutil.Dump(resp)
if err != nil {
return xerrors.Errorf("serializing response: %w", err)
}
@ -92,7 +88,7 @@ func (p *Provider) sendSignedResponse(resp *Response) error {
Signature: sig,
}
err = cborrpc.WriteCborRPC(s, signedResponse)
err = cborutil.WriteCborRPC(s, signedResponse)
if err != nil {
// Assume client disconnected
s.Close()
@ -101,6 +97,17 @@ func (p *Provider) sendSignedResponse(resp *Response) error {
return err
}
func (p *Provider) disconnect(deal MinerDeal) error {
s, ok := p.conns[deal.ProposalCid]
if !ok {
return nil
}
err := s.Close()
delete(p.conns, deal.ProposalCid)
return err
}
func (p *Provider) getWorker(miner address.Address) (address.Address, error) {
getworker := &types.Message{
To: miner,

143
chain/deals/state_store.go
View File

@ -1,143 +0,0 @@
package deals
import (
"bytes"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
"golang.org/x/xerrors"
)
type StateStore struct {
ds datastore.Datastore
}
func (st *StateStore) Begin(i cid.Cid, state interface{}) error {
k := datastore.NewKey(i.String())
has, err := st.ds.Has(k)
if err != nil {
return err
}
if has {
return xerrors.Errorf("Already tracking state for %s", i)
}
b, err := cborrpc.Dump(state)
if err != nil {
return err
}
return st.ds.Put(k, b)
}
func (st *StateStore) End(i cid.Cid) error {
k := datastore.NewKey(i.String())
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
return st.ds.Delete(k)
}
func (st *StateStore) mutate(i cid.Cid, mutator func([]byte) ([]byte, error)) error {
k := datastore.NewKey(i.String())
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
cur, err := st.ds.Get(k)
if err != nil {
return err
}
mutated, err := mutator(cur)
if err != nil {
return err
}
return st.ds.Put(k, mutated)
}
type MinerStateStore struct {
StateStore
}
func (st *MinerStateStore) MutateMiner(i cid.Cid, mutator func(*MinerDeal) error) error {
return st.mutate(i, minerMutator(mutator))
}
func minerMutator(m func(*MinerDeal) error) func([]byte) ([]byte, error) {
return func(in []byte) ([]byte, error) {
deal := new(MinerDeal)
err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal)
if err != nil {
return nil, err
}
if err := m(deal); err != nil {
return nil, err
}
return cborrpc.Dump(deal)
}
}
type ClientStateStore struct {
StateStore
}
func (st *ClientStateStore) MutateClient(i cid.Cid, mutator func(*ClientDeal) error) error {
return st.mutate(i, clientMutator(mutator))
}
func clientMutator(m func(*ClientDeal) error) func([]byte) ([]byte, error) {
return func(in []byte) ([]byte, error) {
deal := new(ClientDeal)
err := cborrpc.ReadCborRPC(bytes.NewReader(in), deal)
if err != nil {
return nil, err
}
if err := m(deal); err != nil {
return nil, err
}
return cborrpc.Dump(deal)
}
}
func (st *ClientStateStore) ListClient() ([]ClientDeal, error) {
var out []ClientDeal
res, err := st.ds.Query(query.Query{})
if err != nil {
return nil, err
}
defer res.Close()
for {
res, ok := res.NextSync()
if !ok {
break
}
var deal ClientDeal
err := cborrpc.ReadCborRPC(bytes.NewReader(res.Value), &deal)
if err != nil {
return nil, err
}
out = append(out, deal)
}
return out, nil
}

21
chain/deals/types.go
View File

@ -5,14 +5,17 @@ import (
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/ipfs/go-cid"
)
const DealProtocolID = "/fil/storage/mk/1.0.0"
const AskProtocolID = "/fil/storage/ask/1.0.0"
const DealProtocolID = "/fil/storage/mk/1.0.1"
const AskProtocolID = "/fil/storage/ask/1.0.1"
type Proposal struct {
DealProposal actors.StorageDealProposal
DealProposal *actors.StorageDealProposal
Piece cid.Cid // Used for retrieving from the client
}
type Response struct {
@ -25,9 +28,6 @@ type Response struct {
// DealAccepted
StorageDeal *actors.StorageDeal
PublishMessage *cid.Cid
// DealComplete
CommitMessage *cid.Cid
}
// TODO: Do we actually need this to be signed?
@ -37,6 +37,15 @@ type SignedResponse struct {
Signature *types.Signature
}
func (r *SignedResponse) Verify(addr address.Address) error {
b, err := cborutil.Dump(&r.Response)
if err != nil {
return err
}
return r.Signature.Verify(addr, b)
}
type AskRequest struct {
Miner address.Address
}

6
chain/events/events.go
View File

@ -18,8 +18,8 @@ import (
var log = logging.Logger("events")
// `curH`-`ts.Height` = `confidence`
type HeightHandler func(ts *types.TipSet, curH uint64) error
type RevertHandler func(ts *types.TipSet) error
type HeightHandler func(ctx context.Context, ts *types.TipSet, curH uint64) error
type RevertHandler func(ctx context.Context, ts *types.TipSet) error
type heightHandler struct {
confidence int
@ -59,6 +59,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events {
heightEvents: heightEvents{
tsc: tsc,
ctx: ctx,
gcConfidence: uint64(gcConfidence),
heightTriggers: map[uint64]*heightHandler{},
@ -69,6 +70,7 @@ func NewEvents(ctx context.Context, api eventApi) *Events {
calledEvents: calledEvents{
cs: api,
tsc: tsc,
ctx: ctx,
gcConfidence: uint64(gcConfidence),
confQueue: map[triggerH]map[msgH][]*queuedEvent{},

17
chain/events/events_called.go
View File

@ -24,7 +24,7 @@ type triggerH = uint64
// `ts` is the tipset, in which the `msg` is included.
// `curH`-`ts.Height` = `confidence`
type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error)
type CalledHandler func(msg *types.Message, ts *types.TipSet, curH uint64) (more bool, err error)
// CheckFunc is used for atomicity guarantees. If the condition the callbacks
// wait for has already happened in tipset `ts`
@ -56,6 +56,7 @@ type queuedEvent struct {
type calledEvents struct {
cs eventApi
tsc *tipSetCache
ctx context.Context
gcConfidence uint64
lk sync.Mutex
@ -114,7 +115,7 @@ func (e *calledEvents) handleReverts(ts *types.TipSet) {
trigger := e.triggers[event.trigger]
if err := trigger.revert(ts); err != nil {
if err := trigger.revert(e.ctx, ts); err != nil {
log.Errorf("reverting chain trigger (call %s.%d() @H %d, called @ %d) failed: %s", event.msg.To, event.msg.Method, ts.Height(), triggerH, err)
}
}
@ -146,7 +147,11 @@ func (e *calledEvents) checkNewCalls(ts *types.TipSet) {
func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message, ts *types.TipSet) {
trigger := e.triggers[triggerId]
triggerH := ts.Height() + uint64(trigger.confidence)
// messages are not applied in the tipset they are included in
appliedH := ts.Height() + 1
triggerH := appliedH + uint64(trigger.confidence)
byOrigH, ok := e.confQueue[triggerH]
if !ok {
@ -154,13 +159,13 @@ func (e *calledEvents) queueForConfidence(triggerId uint64, msg *types.Message,
e.confQueue[triggerH] = byOrigH
}
byOrigH[ts.Height()] = append(byOrigH[ts.Height()], &queuedEvent{
byOrigH[appliedH] = append(byOrigH[appliedH], &queuedEvent{
trigger: triggerId,
h: ts.Height(),
h: appliedH,
msg: msg,
})
e.revertQueue[ts.Height()] = append(e.revertQueue[ts.Height()], triggerH) // todo: dedupe?
e.revertQueue[appliedH] = append(e.revertQueue[appliedH], triggerH)
}
func (e *calledEvents) applyWithConfidence(ts *types.TipSet) {

35
chain/events/events_height.go
View File

@ -1,8 +1,11 @@
package events
import (
"context"
"sync"
"go.opencensus.io/trace"
"github.com/filecoin-project/lotus/chain/types"
)
@ -17,16 +20,29 @@ type heightEvents struct {
htTriggerHeights map[triggerH][]triggerId
htHeights map[msgH][]triggerId
ctx context.Context
}
func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
ctx, span := trace.StartSpan(e.ctx, "events.HeightHeadChange")
defer span.End()
span.AddAttributes(trace.Int64Attribute("endHeight", int64(app[0].Height())))
span.AddAttributes(trace.Int64Attribute("reverts", int64(len(rev))))
span.AddAttributes(trace.Int64Attribute("applies", int64(len(app))))
for _, ts := range rev {
// TODO: log error if h below gcconfidence
// revert height-based triggers
revert := func(h uint64, ts *types.TipSet) {
for _, tid := range e.htHeights[h] {
err := e.heightTriggers[tid].revert(ts)
ctx, span := trace.StartSpan(ctx, "events.HeightRevert")
err := e.heightTriggers[tid].revert(ctx, ts)
span.End()
if err != nil {
log.Errorf("reverting chain trigger (@H %d): %s", h, err)
}
@ -74,7 +90,13 @@ func (e *heightEvents) headChangeAt(rev, app []*types.TipSet) error {
return err
}
if err := hnd.handle(incTs, h); err != nil {
ctx, span := trace.StartSpan(ctx, "events.HeightApply")
span.AddAttributes(trace.BoolAttribute("immediate", false))
err = hnd.handle(ctx, incTs, h)
span.End()
if err != nil {
log.Errorf("chain trigger (@H %d, called @ %d) failed: %s", triggerH, ts.Height(), err)
}
}
@ -125,9 +147,16 @@ func (e *heightEvents) ChainAt(hnd HeightHandler, rev RevertHandler, confidence
}
e.lk.Unlock()
if err := hnd(ts, bestH); err != nil {
ctx, span := trace.StartSpan(e.ctx, "events.HeightApply")
span.AddAttributes(trace.BoolAttribute("immediate", true))
err = hnd(ctx, ts, bestH)
span.End()
if err != nil {
return err
}
e.lk.Lock()
bestH = e.tsc.best().Height()
}

108
chain/events/events_test.go
View File

@ -46,11 +46,24 @@ func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet
func makeTs(t *testing.T, h uint64, msgcid cid.Cid) *types.TipSet {
a, _ := address.NewFromString("t00")
b, _ := address.NewFromString("t02")
ts, err := types.NewTipSet([]*types.BlockHeader{
{
Height: h,
Miner: a,
Tickets: []*types.Ticket{{[]byte{byte(h % 2)}}},
ParentStateRoot: dummyCid,
Messages: msgcid,
ParentMessageReceipts: dummyCid,
},
{
Height: h,
Miner: b,
Tickets: []*types.Ticket{{[]byte{byte((h + 1) % 2)}}},
ParentStateRoot: dummyCid,
Messages: msgcid,
ParentMessageReceipts: dummyCid,
@ -183,12 +196,12 @@ func TestAt(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 5, int(ts.Height()))
require.Equal(t, 8, int(curH))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -248,12 +261,12 @@ func TestAtNullTrigger(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, uint64(6), ts.Height())
require.Equal(t, 8, int(curH))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -282,12 +295,12 @@ func TestAtNullConf(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 5, int(ts.Height()))
require.Equal(t, 8, int(curH))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -323,12 +336,12 @@ func TestAtStart(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 5, int(ts.Height()))
require.Equal(t, 8, int(curH))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -357,12 +370,12 @@ func TestAtStartConfidence(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 5, int(ts.Height()))
require.Equal(t, 11, int(curH))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -385,16 +398,16 @@ func TestAtChained(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
return events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 10, int(ts.Height()))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 10)
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -421,16 +434,16 @@ func TestAtChainedConfidence(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
return events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
return events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
require.Equal(t, 10, int(ts.Height()))
applied = true
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 10)
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -455,11 +468,11 @@ func TestAtChainedConfidenceNull(t *testing.T) {
var applied bool
var reverted bool
err := events.ChainAt(func(ts *types.TipSet, curH uint64) error {
err := events.ChainAt(func(_ context.Context, ts *types.TipSet, curH uint64) error {
applied = true
require.Equal(t, 6, int(ts.Height()))
return nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 5)
@ -494,12 +507,13 @@ func TestCalled(t *testing.T) {
err = events.Called(func(ts *types.TipSet) (d bool, m bool, e error) {
return false, true, nil
}, func(msg *types.Message, ts *types.TipSet, curH uint64) (bool, error) {
require.Equal(t, false, applied)
applied = true
appliedMsg = msg
appliedTs = ts
appliedH = curH
return more, nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
reverted = true
return nil
}, 3, 20, t0123, 5)
@ -526,28 +540,28 @@ func TestCalled(t *testing.T) {
// create additional block so we are above confidence threshold
fcs.advance(0, 1, nil) // H=9 (confidence=3, apply)
fcs.advance(0, 2, nil) // H=10 (confidence=3, apply)
require.Equal(t, true, applied)
require.Equal(t, false, reverted)
applied = false
require.Equal(t, uint64(6), appliedTs.Height())
require.Equal(t, "bafkqaajq", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(9), appliedH)
require.Equal(t, uint64(7), appliedTs.Height())
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(10), appliedH)
require.Equal(t, t0123, appliedMsg.To)
require.Equal(t, uint64(1), appliedMsg.Nonce)
require.Equal(t, uint64(5), appliedMsg.Method)
// revert some blocks, keep the message
fcs.advance(3, 1, nil) // H=7 (confidence=1)
fcs.advance(3, 1, nil) // H=8 (confidence=1)
require.Equal(t, false, applied)
require.Equal(t, false, reverted)
// revert the message
fcs.advance(2, 1, nil) // H=6, we reverted ts with the msg
fcs.advance(2, 1, nil) // H=7, we reverted ts with the msg
require.Equal(t, false, applied)
require.Equal(t, true, reverted)
@ -561,7 +575,7 @@ func TestCalled(t *testing.T) {
},
})
fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=7; H=10 (confidence=3)
fcs.advance(0, 5, map[int]cid.Cid{ // (confidence=3)
0: n2msg,
})
@ -569,16 +583,16 @@ func TestCalled(t *testing.T) {
require.Equal(t, false, reverted)
applied = false
require.Equal(t, uint64(7), appliedTs.Height())
require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(10), appliedH)
require.Equal(t, uint64(9), appliedTs.Height())
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(12), appliedH)
require.Equal(t, t0123, appliedMsg.To)
require.Equal(t, uint64(2), appliedMsg.Nonce)
require.Equal(t, uint64(5), appliedMsg.Method)
// revert and apply at different height
fcs.advance(4, 5, map[int]cid.Cid{ // msg at H=8; H=11 (confidence=3)
fcs.advance(4, 6, map[int]cid.Cid{ // (confidence=3)
1: n2msg,
})
@ -590,16 +604,16 @@ func TestCalled(t *testing.T) {
reverted = false
applied = false
require.Equal(t, uint64(8), appliedTs.Height())
require.Equal(t, "bafkqaajr", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(11), appliedH)
require.Equal(t, uint64(11), appliedTs.Height())
require.Equal(t, "bafkqaaa", appliedTs.Blocks()[0].Messages.String())
require.Equal(t, uint64(14), appliedH)
require.Equal(t, t0123, appliedMsg.To)
require.Equal(t, uint64(2), appliedMsg.Nonce)
require.Equal(t, uint64(5), appliedMsg.Method)
// call method again
fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=12; H=15
fcs.advance(0, 5, map[int]cid.Cid{
0: n2msg,
})
@ -608,7 +622,7 @@ func TestCalled(t *testing.T) {
applied = false
// send and revert below confidence, then cross confidence
fcs.advance(0, 1, map[int]cid.Cid{ // msg at H=16; H=16
fcs.advance(0, 2, map[int]cid.Cid{
0: fcs.fakeMsgs(fakeMsg{
bmsgs: []*types.Message{
{To: t0123, From: t0123, Method: 5, Nonce: 3},
@ -623,7 +637,7 @@ func TestCalled(t *testing.T) {
// test timeout (it's set to 20 in the call to `events.Called` above)
fcs.advance(0, 6, nil) // H=25
fcs.advance(0, 6, nil)
require.Equal(t, false, applied) // not calling timeout as we received messages
require.Equal(t, false, reverted)
@ -631,7 +645,7 @@ func TestCalled(t *testing.T) {
// test unregistering with more
more = false
fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29
fcs.advance(0, 5, map[int]cid.Cid{
0: fcs.fakeMsgs(fakeMsg{
bmsgs: []*types.Message{
{To: t0123, From: t0123, Method: 5, Nonce: 4}, // this signals we don't want more
@ -643,7 +657,7 @@ func TestCalled(t *testing.T) {
require.Equal(t, false, reverted)
applied = false
fcs.advance(0, 4, map[int]cid.Cid{ // msg at H=26; H=29
fcs.advance(0, 5, map[int]cid.Cid{
0: fcs.fakeMsgs(fakeMsg{
bmsgs: []*types.Message{
{To: t0123, From: t0123, Method: 5, Nonce: 5},
@ -693,7 +707,7 @@ func TestCalledTimeout(t *testing.T) {
require.Equal(t, uint64(20), ts.Height())
require.Equal(t, uint64(23), curH)
return false, nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
t.Fatal("revert on timeout")
return nil
}, 3, 20, t0123, 5)
@ -728,7 +742,7 @@ func TestCalledTimeout(t *testing.T) {
require.Equal(t, uint64(20), ts.Height())
require.Equal(t, uint64(23), curH)
return false, nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
t.Fatal("revert on timeout")
return nil
}, 3, 20, t0123, 5)
@ -765,21 +779,21 @@ func TestCalledOrder(t *testing.T) {
switch at {
case 0:
require.Equal(t, uint64(1), msg.Nonce)
require.Equal(t, uint64(3), ts.Height())
require.Equal(t, uint64(4), ts.Height())
case 1:
require.Equal(t, uint64(2), msg.Nonce)
require.Equal(t, uint64(4), ts.Height())
require.Equal(t, uint64(5), ts.Height())
default:
t.Fatal("apply should only get called twice, at: ", at)
}
at++
return true, nil
}, func(ts *types.TipSet) error {
}, func(_ context.Context, ts *types.TipSet) error {
switch at {
case 2:
require.Equal(t, uint64(4), ts.Height())
require.Equal(t, uint64(5), ts.Height())
case 3:
require.Equal(t, uint64(3), ts.Height())
require.Equal(t, uint64(4), ts.Height())
default:
t.Fatal("revert should only get called twice, at: ", at)
}

79
chain/market/fundmgr.go Normal file
View File

@ -0,0 +1,79 @@
package market
import (
"context"
"sync"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl/full"
)
type FundMgr struct {
sm *stmgr.StateManager
mpool full.MpoolAPI
lk sync.Mutex
available map[address.Address]types.BigInt
}
func NewFundMgr(sm *stmgr.StateManager, mpool full.MpoolAPI) *FundMgr {
return &FundMgr{
sm: sm,
mpool: mpool,
available: map[address.Address]types.BigInt{},
}
}
func (fm *FundMgr) EnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error {
fm.lk.Lock()
avail, ok := fm.available[addr]
if !ok {
bal, err := fm.sm.MarketBalance(ctx, addr, nil)
if err != nil {
fm.lk.Unlock()
return err
}
avail = bal.Available
}
toAdd := types.NewInt(0)
avail = types.BigSub(avail, amt)
if avail.LessThan(types.NewInt(0)) {
// TODO: some rules around adding more to avoid doing stuff on-chain
// all the time
toAdd = types.BigSub(toAdd, avail)
avail = types.NewInt(0)
}
fm.available[addr] = avail
fm.lk.Unlock()
smsg, err := fm.mpool.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: addr,
Value: toAdd,
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.AddBalance,
})
if err != nil {
return err
}
_, r, err := fm.sm.WaitForMessage(ctx, smsg.Cid())
if err != nil {
return err
}
if r.ExitCode != 0 {
return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.ExitCode)
}
return nil
}

30
chain/stmgr/utils.go
View File

@ -157,7 +157,7 @@ func GetMinerProvingPeriodEnd(ctx context.Context, sm *StateManager, ts *types.T
return mas.ProvingPeriodEnd, nil
}
func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) {
func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
@ -167,7 +167,7 @@ func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet,
return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.ProvingSet)
}
func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.SectorInfo, error) {
func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
@ -193,20 +193,40 @@ func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet,
return minfo.SectorSize, nil
}
func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.SectorInfo, error) {
func GetStorageDeal(ctx context.Context, sm *StateManager, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) {
var state actors.StorageMarketState
if _, err := sm.LoadActorState(ctx, actors.StorageMarketAddress, &state, ts); err != nil {
return nil, err
}
blks := amt.WrapBlockstore(sm.ChainStore().Blockstore())
da, err := amt.LoadAMT(blks, state.Deals)
if err != nil {
return nil, err
}
var ocd actors.OnChainDeal
if err := da.Get(dealId, &ocd); err != nil {
return nil, err
}
return &ocd, nil
}
func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.Cid) ([]*api.ChainSectorInfo, error) {
blks := amt.WrapBlockstore(bs)
a, err := amt.LoadAMT(blks, ssc)
if err != nil {
return nil, err
}
var sset []*api.SectorInfo
var sset []*api.ChainSectorInfo
if err := a.ForEach(func(i uint64, v *cbg.Deferred) error {
var comms [][]byte
if err := cbor.DecodeInto(v.Raw, &comms); err != nil {
return err
}
sset = append(sset, &api.SectorInfo{
sset = append(sset, &api.ChainSectorInfo{
SectorID: i,
CommR: comms[0],
CommD: comms[1],

2
chain/sub/incoming.go
View File

@ -44,7 +44,7 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
return
}
log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
s.InformNewBlock(msg.GetFrom(), &types.FullBlock{
Header: blk.Header,
BlsMessages: bmsgs,

28
chain/sync.go
View File

@ -2,6 +2,7 @@ package chain
import (
"context"
"errors"
"fmt"
"sync"
"time"
@ -102,7 +103,7 @@ func (syncer *Syncer) InformNewHead(from peer.ID, fts *store.FullTipSet) {
if from == syncer.self {
// TODO: this is kindof a hack...
log.Info("got block from ourselves")
log.Debug("got block from ourselves")
if err := syncer.Sync(ctx, fts.TipSet()); err != nil {
log.Errorf("failed to sync our own block %s: %+v", fts.TipSet().Cids(), err)
@ -368,6 +369,10 @@ func (syncer *Syncer) Sync(ctx context.Context, maybeHead *types.TipSet) error {
return nil
}
func isPermanent(err error) bool {
return !errors.Is(err, ErrTemporal)
}
func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error {
ctx, span := trace.StartSpan(ctx, "validateTipSet")
defer span.End()
@ -379,7 +384,9 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet)
for _, b := range fts.Blocks {
if err := syncer.ValidateBlock(ctx, b); err != nil {
syncer.bad.Add(b.Cid())
if isPermanent(err) {
syncer.bad.Add(b.Cid())
}
return xerrors.Errorf("validating block %s: %w", b.Cid(), err)
}
@ -444,6 +451,8 @@ func (syncer *Syncer) validateTickets(ctx context.Context, mworker address.Addre
return nil
}
var ErrTemporal = errors.New("temporal error")
// Should match up with 'Semantical Validation' in validation.md in the spec
func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error {
ctx, span := trace.StartSpan(ctx, "validateBlock")
@ -462,6 +471,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
}
if stateroot != h.ParentStateRoot {
msgs, err := syncer.store.MessagesForTipset(baseTs)
if err != nil {
log.Error("failed to load messages for tipset during tipset state mismatch error: ", err)
} else {
log.Warn("Messages for tipset with mismatching state:")
for i, m := range msgs {
mm := m.VMMessage()
log.Warnf("Message[%d]: from=%s to=%s method=%d params=%x", i, mm.From, mm.To, mm.Method, mm.Params)
}
}
return xerrors.Errorf("parent state root did not match computed state (%s != %s)", stateroot, h.ParentStateRoot)
}
@ -470,7 +490,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
}
if h.Timestamp > uint64(time.Now().Unix()+build.AllowableClockDrift) {
return xerrors.Errorf("block was from the future")
return xerrors.Errorf("block was from the future: %w", ErrTemporal)
}
if h.Timestamp < baseTs.MinTimestamp()+uint64(build.BlockDelay*len(h.Tickets)) {
@ -907,7 +927,7 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
}
syncer.syncState.SetStage(api.StageSyncComplete)
log.Infow("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
log.Debugw("new tipset", "height", ts.Height(), "tipset", types.LogCids(ts.Cids()))
return nil
}

3
chain/sync_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/store"
@ -390,7 +391,7 @@ func TestSyncBadTimestamp(t *testing.T) {
base := tu.g.CurTipset
tu.g.Timestamper = func(pts *types.TipSet, tl int) uint64 {
return pts.MinTimestamp() + 2
return pts.MinTimestamp() + (build.BlockDelay / 2)
}
fmt.Println("BASE: ", base.Cids())

17
chain/types/bigint_test.go
View File

@ -32,3 +32,20 @@ func TestBigIntSerializationRoundTrip(t *testing.T) {
}
}
func TestFilRoundTrip(t *testing.T) {
testValues := []string{
"0", "1", "1.001", "100.10001", "101100", "5000.01", "5000",
}
for _, v := range testValues {
fval, err := ParseFIL(v)
if err != nil {
t.Fatal(err)
}
if fval.String() != v {
t.Fatal("mismatch in values!", v, fval.String())
}
}
}

115
chain/types/cbor_gen.go
View File

@ -1470,3 +1470,118 @@ func (t *StorageAsk) UnmarshalCBOR(r io.Reader) error {
t.SeqNo = uint64(extra)
return nil
}
func (t *ExpTipSet) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.t.Cids ([]cid.Cid) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Cids)))); err != nil {
return err
}
for _, v := range t.Cids {
if err := cbg.WriteCid(w, v); err != nil {
return xerrors.Errorf("failed writing cid field t.Cids: %w", err)
}
}
// t.t.Blocks ([]*types.BlockHeader) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Blocks)))); err != nil {
return err
}
for _, v := range t.Blocks {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.t.Height (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Height))); err != nil {
return err
}
return nil
}
func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Cids ([]cid.Cid) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Cids: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Cids = make([]cid.Cid, extra)
}
for i := 0; i < int(extra); i++ {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("reading cid field t.Cids failed: %w", err)
}
t.Cids[i] = c
}
// t.t.Blocks ([]*types.BlockHeader) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Blocks: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Blocks = make([]*BlockHeader, extra)
}
for i := 0; i < int(extra); i++ {
var v BlockHeader
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Blocks[i] = &v
}
// t.t.Height (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Height = uint64(extra)
return nil
}

5
chain/types/fil.go
View File

@ -12,7 +12,10 @@ type FIL BigInt
func (f FIL) String() string {
r := new(big.Rat).SetFrac(f.Int, big.NewInt(build.FilecoinPrecision))
return strings.TrimRight(r.FloatString(18), "0.")
if r.Sign() == 0 {
return "0"
}
return strings.TrimRight(strings.TrimRight(r.FloatString(18), "0"), ".")
}
func (f FIL) Format(s fmt.State, ch rune) {

36
chain/types/tipset.go
View File

@ -4,10 +4,12 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"sort"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log"
cbg "github.com/whyrusleeping/cbor-gen"
)
var log = logging.Logger("types")
@ -20,14 +22,14 @@ type TipSet struct {
// why didnt i just export the fields? Because the struct has methods with the
// same names already
type expTipSet struct {
type ExpTipSet struct {
Cids []cid.Cid
Blocks []*BlockHeader
Height uint64
}
func (ts *TipSet) MarshalJSON() ([]byte, error) {
return json.Marshal(expTipSet{
return json.Marshal(ExpTipSet{
Cids: ts.cids,
Blocks: ts.blks,
Height: ts.height,
@ -35,7 +37,7 @@ func (ts *TipSet) MarshalJSON() ([]byte, error) {
}
func (ts *TipSet) UnmarshalJSON(b []byte) error {
var ets expTipSet
var ets ExpTipSet
if err := json.Unmarshal(b, &ets); err != nil {
return err
}
@ -50,6 +52,34 @@ func (ts *TipSet) UnmarshalJSON(b []byte) error {
return nil
}
func (ts *TipSet) MarshalCBOR(w io.Writer) error {
if ts == nil {
_, err := w.Write(cbg.CborNull)
return err
}
return (&ExpTipSet{
Cids: ts.cids,
Blocks: ts.blks,
Height: ts.height,
}).MarshalCBOR(w)
}
func (ts *TipSet) UnmarshalCBOR(r io.Reader) error {
var ets ExpTipSet
if err := ets.UnmarshalCBOR(r); err != nil {
return err
}
ots, err := NewTipSet(ets.Blocks)
if err != nil {
return err
}
*ts = *ots
return nil
}
func tipsetSortFunc(blks []*BlockHeader) func(i, j int) bool {
return func(i, j int) bool {
ti := blks[i].LastTicket()

2
chain/types/voucher.go
View File

@ -5,7 +5,7 @@ import (
"encoding/base64"
"github.com/filecoin-project/lotus/chain/address"
cborrpc "github.com/filecoin-project/lotus/lib/cborrpc"
cborrpc "github.com/filecoin-project/lotus/lib/cborutil"
cbor "github.com/ipfs/go-ipld-cbor"
)

4
chain/vm/invoker.go
View File

@ -2,6 +2,7 @@ package vm
import (
"bytes"
"encoding/hex"
"fmt"
"reflect"
@ -43,7 +44,8 @@ func (inv *invoker) Invoke(act *types.Actor, vmctx types.VMContext, method uint6
code, ok := inv.builtInCode[act.Code]
if !ok {
return nil, aerrors.Newf(255, "no code for actor %s", act.Code)
log.Errorf("no code for actor %s", act.Code)
return nil, aerrors.Newf(255, "no code for actor %s(%d)(%s)", act.Code, method, hex.EncodeToString(params))
}
if method >= uint64(len(code)) || code[method] == nil {
return nil, aerrors.Newf(255, "no method %d on actor", method)

2
chain/vm/vm.go
View File

@ -462,7 +462,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet,
return nil, xerrors.Errorf("fatal error: %w", actorErr)
}
if actorErr != nil {
log.Warnf("[from=%s,n=%d,h=%d] Send actor error: %+v", msg.From, msg.Nonce, vm.blockHeight, actorErr)
log.Warnf("[from=%s,to=%s,n=%d,m=%d,h=%d] Send actor error: %+v", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr)
}
var errcode uint8

31
cli/client.go
View File

@ -2,14 +2,17 @@ package cli
import (
"fmt"
"os"
"path/filepath"
"strconv"
"text/tabwriter"
"github.com/ipfs/go-cid"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
lapi "github.com/filecoin-project/lotus/api"
actors "github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
@ -25,6 +28,7 @@ var clientCmd = &cli.Command{
clientFindCmd,
clientRetrieveCmd,
clientQueryAskCmd,
clientListDeals,
},
}
@ -308,7 +312,7 @@ var clientQueryAskCmd = &cli.Command{
}
fmt.Printf("Ask: %s\n", maddr)
fmt.Printf("Price per Byte: %s\n", types.FIL(ask.Ask.Price))
fmt.Printf("Price per GigaByte: %s\n", types.FIL(ask.Ask.Price))
size := cctx.Int64("size")
if size == 0 {
@ -326,3 +330,28 @@ var clientQueryAskCmd = &cli.Command{
return nil
},
}
var clientListDeals = &cli.Command{
Name: "list-deals",
Usage: "List storage market deals",
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
deals, err := api.ClientListDeals(ctx)
if err != nil {
return err
}
w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0)
fmt.Fprintf(w, "DealCid\tProvider\tState\tPieceRef\tSize\tPrice\tDuration\n")
for _, d := range deals {
fmt.Fprintf(w, "%s\t%s\t%s\t%x\t%d\t%s\t%d\n", d.ProposalCid, d.Provider, lapi.DealStates[d.State], d.PieceRef, d.Size, d.PricePerEpoch, d.Duration)
}
return w.Flush()
},
}

8
cli/params.go
View File

@ -9,8 +9,14 @@ import (
var fetchParamCmd = &cli.Command{
Name: "fetch-params",
Usage: "Fetch proving parameters",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "only-verify-keys",
Usage: "only download the verify keys",
},
},
Action: func(cctx *cli.Context) error {
if err := build.GetParams(true); err != nil {
if err := build.GetParams(!cctx.Bool("only-verify-keys")); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}

24
cmd/lotus-fountain/main.go
View File

@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"net"
"net/http"
"os"
"strconv"
@ -24,7 +25,7 @@ import (
var log = logging.Logger("main")
var sendPerRequest = types.NewInt(500_000_000)
var sendPerRequest, _ = types.ParseFIL("0.005")
func main() {
logging.SetLogLevel("*", "INFO")
@ -104,7 +105,7 @@ var runCmd = &cli.Command{
TotalRate: time.Second,
TotalBurst: 20,
IPRate: 10 * time.Minute,
IPBurst: 1,
IPBurst: 2,
WalletRate: 1 * time.Hour,
WalletBurst: 1,
}),
@ -151,7 +152,20 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
}
// Limit based on IP
limiter = h.limiter.GetIPLimiter(r.RemoteAddr)
reqIP := r.Header.Get("X-Real-IP")
if reqIP == "" {
h, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
log.Errorf("could not get ip from: %s, err: %s", r.RemoteAddr, err)
}
reqIP = h
}
if i := net.ParseIP(reqIP); i != nil && i.IsLoopback() {
log.Errorf("rate limiting localhost: %s", reqIP)
}
limiter = h.limiter.GetIPLimiter(reqIP)
if !limiter.Allow() {
http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
return
@ -164,7 +178,7 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
}
smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
Value: sendPerRequest,
Value: types.BigInt(sendPerRequest),
From: h.from,
To: to,
@ -229,7 +243,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
}
smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
Value: sendPerRequest,
Value: types.BigInt(sendPerRequest),
From: h.from,
To: owner,

41
cmd/lotus-storage-miner/commitments.go
View File

@ -1,41 +0,0 @@
package main
import (
"fmt"
lcli "github.com/filecoin-project/lotus/cli"
"gopkg.in/urfave/cli.v2"
)
var commitmentsCmd = &cli.Command{
Name: "commitments",
Usage: "interact with commitment tracker",
Subcommands: []*cli.Command{
commitmentsListCmd,
},
}
var commitmentsListCmd = &cli.Command{
Name: "list",
Usage: "List tracked sector commitments",
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
comms, err := api.CommitmentsList(ctx)
if err != nil {
return err
}
for _, comm := range comms {
fmt.Printf("%s:%d msg:%s, deals: %v\n", comm.Miner, comm.SectorID, comm.CommitMsg, comm.DealIDs)
}
return nil
},
}

58
cmd/lotus-storage-miner/info.go
View File

@ -6,7 +6,6 @@ import (
"gopkg.in/urfave/cli.v2"
sectorstate "github.com/filecoin-project/go-sectorbuilder/sealing_state"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
@ -37,6 +36,14 @@ var infoCmd = &cli.Command{
fmt.Printf("Miner: %s\n", maddr)
// Sector size
sizeByte, err := api.StateMinerSectorSize(ctx, maddr, nil)
if err != nil {
return err
}
fmt.Printf("Sector Size: %s\n", sizeStr(sizeByte))
pow, err := api.StateMinerPower(ctx, maddr, nil)
if err != nil {
return err
@ -45,40 +52,47 @@ var infoCmd = &cli.Command{
percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower)
fmt.Printf("Power: %s / %s (%0.2f%%)\n", pow.MinerPower, pow.TotalPower, float64(percI.Int64())/1000*100)
// TODO: indicate whether the post worker is in use
wstat, err := nodeApi.WorkerStats(ctx)
if err != nil {
return err
}
fmt.Printf("Worker use: %d / %d (+%d)", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved)
sinfo, err := sectorsInfo(ctx, nodeApi)
if err != nil {
return err
}
fmt.Println("Sealed Sectors:\t", sinfo.SealedCount)
fmt.Println("Sealing Sectors:\t", sinfo.SealingCount)
fmt.Println("Pending Sectors:\t", sinfo.PendingCount)
fmt.Println("Failed Sectors:\t", sinfo.FailedCount)
fmt.Println(sinfo)
// TODO: grab actr state / info
// * Sector size
// * Sealed sectors (count / bytes)
// * Power
return nil
},
}
type SectorsInfo struct {
TotalCount int
SealingCount int
FailedCount int
SealedCount int
PendingCount int
var Units = []string{"B", "KiB", "MiB", "GiB", "TiB"}
func sizeStr(size uint64) string {
i := 0
unitSize := float64(size)
for unitSize >= 1024 && i < len(Units)-1 {
unitSize = unitSize / 1024
i++
}
return fmt.Sprintf("%g %s", unitSize, Units[i])
}
func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, error) {
func sectorsInfo(ctx context.Context, napi api.StorageMiner) (map[string]int, error) {
sectors, err := napi.SectorsList(ctx)
if err != nil {
return nil, err
}
out := SectorsInfo{
TotalCount: len(sectors),
out := map[string]int{
"Total": len(sectors),
}
for _, s := range sectors {
st, err := napi.SectorsStatus(ctx, s)
@ -86,18 +100,8 @@ func sectorsInfo(ctx context.Context, napi api.StorageMiner) (*SectorsInfo, erro
return nil, err
}
switch st.State {
case sectorstate.Sealed:
out.SealedCount++
case sectorstate.Pending:
out.PendingCount++
case sectorstate.Sealing:
out.SealingCount++
case sectorstate.Failed:
out.FailedCount++
case sectorstate.Unknown:
}
out[api.SectorStateStr(st.State)]++
}
return &out, nil
return out, nil
}

1
cmd/lotus-storage-miner/main.go
View File

@ -26,7 +26,6 @@ func main() {
infoCmd,
storeGarbageCmd,
sectorsCmd,
commitmentsCmd,
}
jaeger := tracing.SetupJaegerTracing("lotus")
defer func() {

13
cmd/lotus-storage-miner/run.go
View File

@ -8,13 +8,12 @@ import (
"os/signal"
"syscall"
"github.com/filecoin-project/lotus/build"
"github.com/multiformats/go-multiaddr"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/auth"
"github.com/filecoin-project/lotus/lib/jsonrpc"
@ -32,12 +31,20 @@ var runCmd = &cli.Command{
Name: "api",
Value: "2345",
},
&cli.BoolFlag{
Name: "enable-gpu-proving",
Usage: "Enable use of GPU for mining operations",
},
},
Action: func(cctx *cli.Context) error {
if err := build.GetParams(true); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
if !cctx.Bool("enable-gpu-proving") {
os.Setenv("BELLMAN_NO_GPU", "true")
}
nodeApi, ncloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
@ -77,7 +84,7 @@ var runCmd = &cli.Command{
}
return lr.SetAPIEndpoint(apima)
}),
node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(storageRepoPath)),
node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(storageRepoPath, 5)), // TODO: grab worker count from config
node.Override(new(api.FullNode), nodeApi),
)
if err != nil {

8
cmd/lotus-storage-miner/sectors.go
View File

@ -6,6 +6,7 @@ import (
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/api"
lcli "github.com/filecoin-project/lotus/cli"
)
@ -60,14 +61,15 @@ var sectorsStatusCmd = &cli.Command{
}
fmt.Printf("SectorID:\t%d\n", status.SectorID)
fmt.Printf("Status:\t%s\n", status.State.String())
fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg)
fmt.Printf("Status:\t%s\n", api.SectorStateStr(status.State))
fmt.Printf("CommD:\t\t%x\n", status.CommD)
fmt.Printf("CommR:\t\t%x\n", status.CommR)
fmt.Printf("Ticket:\t\t%x\n", status.Ticket.TicketBytes)
fmt.Printf("TicketH:\t\t%d\n", status.Ticket.BlockHeight)
fmt.Printf("Seed:\t\t%x\n", status.Seed.TicketBytes)
fmt.Printf("SeedH:\t\t%d\n", status.Seed.BlockHeight)
fmt.Printf("Proof:\t\t%x\n", status.Proof)
fmt.Printf("Pieces:\t\t%v\n", status.Pieces)
fmt.Printf("Deals:\t\t%v\n", status.Deals)
return nil
},
}

2
extern/go-bls-sigs vendored

@ -1 +1 @@
Subproject commit 10fef3cbfa8670d9e9cdffb0cd8dfd4f6f6fef07
Subproject commit 98479d3c79620f18783da0c2c6a15f8b8eb4fa2e

2
extern/go-sectorbuilder vendored

@ -1 +1 @@
Subproject commit 692725ff21919ce9c9df9ea87621b0c1e6a9746c
Subproject commit e198d9050b5dde6c19bef3593506597afe00b6cb

20
gen/main.go
View File

@ -12,6 +12,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storage"
)
func main() {
@ -29,6 +30,7 @@ func main() {
types.BlockMsg{},
types.SignedStorageAsk{},
types.StorageAsk{},
types.ExpTipSet{},
)
if err != nil {
fmt.Println(err)
@ -46,6 +48,8 @@ func main() {
err = gen.WriteTupleEncodersToFile("./api/cbor_gen.go", "api",
api.PaymentInfo{},
api.SealedRef{},
api.SealedRefs{},
)
if err != nil {
fmt.Println(err)
@ -85,7 +89,8 @@ func main() {
actors.AccountActorState{},
actors.StorageMinerActorState{},
actors.StorageMinerConstructorParams{},
actors.OnChainSealVerifyInfo{},
actors.SectorPreCommitInfo{},
actors.PreCommittedSector{},
actors.MinerInfo{},
actors.SubmitPoStParams{},
actors.PaymentVerifyParams{},
@ -122,6 +127,8 @@ func main() {
actors.ActivateStorageDealsParams{},
actors.ProcessStorageDealsPaymentParams{},
actors.OnChainDeal{},
actors.ComputeDataCommitmentParams{},
actors.SectorProveCommitInfo{},
)
if err != nil {
fmt.Println(err)
@ -142,4 +149,15 @@ func main() {
fmt.Println(err)
os.Exit(1)
}
err = gen.WriteTupleEncodersToFile("./storage/cbor_gen.go", "storage",
storage.SealTicket{},
storage.SealSeed{},
storage.Piece{},
storage.SectorInfo{},
)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}

2
go.mod
View File

@ -73,7 +73,7 @@ require (
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
github.com/stretchr/testify v1.4.0
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f
github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
go.opencensus.io v0.22.0

142
go.sum
View File

@ -13,10 +13,8 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo
github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo=
@ -30,8 +28,6 @@ github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
@ -75,7 +71,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA=
@ -88,35 +83,16 @@ github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@ -126,29 +102,11 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.18.0/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
@ -163,7 +121,6 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@ -174,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@ -291,8 +247,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M
github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@ -300,13 +254,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ=
github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
@ -433,10 +382,8 @@ github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw
github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI=
github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI=
github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
@ -445,7 +392,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@ -453,7 +401,6 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@ -465,17 +412,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+
github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM=
github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78=
@ -503,30 +442,22 @@ github.com/multiformats/go-multihash v0.0.7/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA
github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ=
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c/go.mod h1:eTBvSIlRgLo+CNFFQRQTwUGTZOEdvXIKeZS/xG+D2yU=
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg=
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229 h1:E2B8qYyeSgv5MXpmzZXRNp8IAQ4vjxIjhpAf5hv/tAg=
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -541,32 +472,16 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
@ -576,27 +491,17 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:s
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81/go.mod h1:xIvvI5FiHLxhv8prbzVpaMHaaGPFPFQSuTcxC91ryOo=
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
github.com/sourcegraph/go-langserver v2.0.0+incompatible/go.mod h1:bBMjfpzEHd6ijPRoQ7f+knFfw+e8R+W158/MsqAy77c=
github.com/sourcegraph/jsonrpc2 v0.0.0-20190106185902-35a74f039c6a/go.mod h1:eESpbCslcLDs8j2D7IEdGVgul7xuk9odqDTaor30IUU=
github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU=
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -606,19 +511,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0=
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
@ -628,8 +528,8 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba h1:X4n8JG2e2
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM=
github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f h1:+GFA37QICd1Axd2n9uzjtvPjxJJI5PU78vpvam+hI4U=
github.com/whyrusleeping/cbor-gen v0.0.0-20191104210213-4418c8842f0f/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679 h1:ct50KYdZHcdOnTAuSgppw5MZKTa3RA63FX28m0l9Aeg=
github.com/whyrusleeping/cbor-gen v0.0.0-20191107223350-6fdade89d679/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@ -676,13 +576,11 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -692,11 +590,9 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -710,10 +606,11 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -723,7 +620,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -736,7 +632,6 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -744,11 +639,11 @@ golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@ -757,26 +652,17 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190813142322-97f12d73768f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -797,27 +683,27 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk=
gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y=
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@ -826,7 +712,3 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

14
lib/cborrpc/rpc.go → lib/cborutil/rpc.go
View File

@ -1,4 +1,4 @@
package cborrpc
package cborutil
import (
"bytes"
@ -66,3 +66,15 @@ func AsIpld(obj interface{}) (ipld.Node, error) {
}
return cbor.WrapObject(obj, math.MaxUint64, -1)
}
func Equals(a cbg.CBORMarshaler, b cbg.CBORMarshaler) (bool, error) {
ab, err := Dump(a)
if err != nil {
return false, err
}
bb, err := Dump(b)
if err != nil {
return false, err
}
return bytes.Equal(ab, bb), nil
}

4
lib/jsonrpc/client.go
View File

@ -177,9 +177,9 @@ func (c *client) makeOutChan(ctx context.Context, ftyp reflect.Type, valOut int)
front := buf.Front()
bufLk.Unlock()
cases := []reflect.SelectCase{
cases := []reflect.SelectCase{
{
Dir: reflect.SelectRecv,
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(chCtx.Done()),
},
{

38
lib/padreader/padreader.go Normal file
View File

@ -0,0 +1,38 @@
package padreader
import (
"io"
"math/bits"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
)
func PaddedSize(size uint64) uint64 {
logv := 64 - bits.LeadingZeros64(size)
sectSize := uint64(1 << logv)
bound := sectorbuilder.GetMaxUserBytesPerStagedSector(sectSize)
if size <= bound {
return bound
}
return sectorbuilder.GetMaxUserBytesPerStagedSector(1 << (logv + 1))
}
type nullReader struct{}
func (nr nullReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
return len(b), nil
}
func New(r io.Reader, size uint64) (io.Reader, uint64) {
padSize := PaddedSize(size)
return io.MultiReader(
io.LimitReader(r, int64(size)),
io.LimitReader(nullReader{}, int64(padSize-size)),
), padSize
}

18
lib/padreader/padreader_test.go Normal file
View File

@ -0,0 +1,18 @@
package padreader
import (
"gotest.tools/assert"
"testing"
)
func TestComputePaddedSize(t *testing.T) {
assert.Equal(t, uint64(1040384), PaddedSize(1000000))
assert.Equal(t, uint64(1016), PaddedSize(548))
assert.Equal(t, uint64(1016), PaddedSize(1015))
assert.Equal(t, uint64(1016), PaddedSize(1016))
assert.Equal(t, uint64(2032), PaddedSize(1017))
assert.Equal(t, uint64(2032), PaddedSize(1024))
assert.Equal(t, uint64(4064), PaddedSize(2048))
}

88
lib/sectorbuilder/files.go Normal file
View File

@ -0,0 +1,88 @@
package sectorbuilder
import (
"fmt"
"io"
"os"
"path/filepath"
"sync"
"golang.org/x/xerrors"
)
func (sb *SectorBuilder) sectorName(sectorID uint64) string {
return fmt.Sprintf("s-%s-%d", sb.Miner, sectorID)
}
func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string {
return filepath.Join(sb.stagedDir, sb.sectorName(sectorID))
}
func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
}
func (sb *SectorBuilder) sealedSectorPath(sectorID uint64) (string, error) {
path := filepath.Join(sb.sealedDir, sb.sectorName(sectorID))
e, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return "", err
}
return path, e.Close()
}
func (sb *SectorBuilder) sectorCacheDir(sectorID uint64) (string, error) {
dir := filepath.Join(sb.cacheDir, sb.sectorName(sectorID))
err := os.Mkdir(dir, 0755)
if os.IsExist(err) {
err = nil
}
return dir, err
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var wait sync.Mutex
var werr error
wait.Lock()
go func() {
defer wait.Unlock()
copied, werr := io.CopyN(w, r, n)
if werr != nil {
log.Warnf("toReadableFile: copy error: %+v", werr)
}
err := w.Close()
if werr == nil && err != nil {
werr = err
log.Warnf("toReadableFile: close error: %+v", err)
return
}
if copied != n {
log.Warnf("copied different amount than expected: %d != %d", copied, n)
werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n)
}
}()
return f, func() error {
wait.Lock()
return werr
}, nil
}

37
lib/sectorbuilder/mock.go
View File

@ -6,37 +6,50 @@ import (
"path/filepath"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func TempSectorbuilder(sectorSize uint64) (*SectorBuilder, func(), error) {
func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, func(), error) {
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
return nil, nil, err
}
sb, err := TempSectorbuilderDir(dir, sectorSize, ds)
return sb, func() {
sb.Destroy()
if err := os.RemoveAll(dir); err != nil {
log.Warn("failed to clean up temp sectorbuilder: ", err)
}
}, err
}
func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder, error) {
addr, err := address.NewFromString("t3vfxagwiegrywptkbmyohqqbfzd7xzbryjydmxso4hfhgsnv6apddyihltsbiikjf3lm7x2myiaxhuc77capq")
if err != nil {
return nil, nil, err
return nil, err
}
metadata := filepath.Join(dir, "meta")
sealed := filepath.Join(dir, "sealed")
staging := filepath.Join(dir, "staging")
cache := filepath.Join(dir, "cache")
sb, err := New(&Config{
SectorSize: sectorSize,
sb, err := New(&SectorBuilderConfig{
SectorSize: sectorSize,
SealedDir: sealed,
StagedDir: staging,
MetadataDir: metadata,
Miner: addr,
})
CacheDir: cache,
WorkerThreads: 2,
Miner: addr,
}, ds)
if err != nil {
return nil, nil, err
return nil, err
}
return sb, func() {
if err := os.RemoveAll(dir); err != nil {
log.Warn("failed to clean up temp sectorbuilder: ", err)
}
}, nil
return sb, nil
}

305
lib/sectorbuilder/sectorbuilder.go
View File

@ -1,19 +1,28 @@
package sectorbuilder
import (
"fmt"
"io"
"os"
"sort"
"strconv"
"sync"
"unsafe"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
const PoStReservedWorkers = 1
const PoRepProofPartitions = 2
var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
var log = logging.Logger("sectorbuilder")
type SectorSealingStatus = sectorbuilder.SectorSealingStatus
@ -26,36 +35,111 @@ type SectorInfo = sectorbuilder.SectorInfo
type SealTicket = sectorbuilder.SealTicket
type SealedSectorMetadata = sectorbuilder.SealedSectorMetadata
type SealSeed = sectorbuilder.SealSeed
type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput
type SealCommitOutput = sectorbuilder.SealCommitOutput
type PublicPieceInfo = sectorbuilder.PublicPieceInfo
type RawSealPreCommitOutput = sectorbuilder.RawSealPreCommitOutput
const CommLen = sectorbuilder.CommitmentBytesLen
type SectorBuilder struct {
handle unsafe.Pointer
ds dtypes.MetadataDS
idLk sync.Mutex
ssize uint64
Miner address.Address
stagedDir string
sealedDir string
cacheDir string
rateLimit chan struct{}
}
type SectorBuilderConfig struct {
SectorSize uint64
Miner address.Address
type Config struct {
SectorSize uint64
Miner address.Address
WorkerThreads uint8
CacheDir string
SealedDir string
StagedDir string
MetadataDir string
}
func New(cfg *SectorBuilderConfig) (*SectorBuilder, error) {
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
if cfg.WorkerThreads <= PoStReservedWorkers {
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads)
}
proverId := addressToProverID(cfg.Miner)
sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, 2, 1, 0, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, 16)
for _, dir := range []string{cfg.StagedDir, cfg.SealedDir, cfg.CacheDir, cfg.MetadataDir} {
if err := os.Mkdir(dir, 0755); err != nil {
if os.IsExist(err) {
continue
}
return nil, err
}
}
var lastUsedID uint64
b, err := ds.Get(lastSectorIdKey)
switch err {
case nil:
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
return nil, err
}
lastUsedID = uint64(i)
case datastore.ErrNotFound:
default:
return nil, err
}
sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, lastUsedID, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads)
if err != nil {
return nil, err
}
return &SectorBuilder{
sb := &SectorBuilder{
handle: sbp,
Miner: cfg.Miner,
}, nil
ds: ds,
ssize: cfg.SectorSize,
stagedDir: cfg.StagedDir,
sealedDir: cfg.SealedDir,
cacheDir: cfg.CacheDir,
Miner: cfg.Miner,
rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers),
}
return sb, nil
}
func (sb *SectorBuilder) rlimit() func() {
if cap(sb.rateLimit) == len(sb.rateLimit) {
log.Warn("rate-limiting sectorbuilder call")
}
sb.rateLimit <- struct{}{}
return func() {
<-sb.rateLimit
}
}
func (sb *SectorBuilder) WorkerStats() (free, reserved, total int) {
return cap(sb.rateLimit) - len(sb.rateLimit), PoStReservedWorkers, cap(sb.rateLimit) + PoStReservedWorkers
}
func addressToProverID(a address.Address) [32]byte {
@ -68,31 +152,161 @@ func (sb *SectorBuilder) Destroy() {
sectorbuilder.DestroySectorBuilder(sb.handle)
}
func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Reader) (uint64, error) {
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
sb.idLk.Lock()
defer sb.idLk.Unlock()
id, err := sectorbuilder.AcquireSectorId(sb.handle)
if err != nil {
return 0, err
}
err = sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
if err != nil {
return 0, err
}
return id, nil
}
func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) {
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return 0, err
return PublicPieceInfo{}, err
}
sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f)
ret := sb.rlimit()
defer ret()
stagedFile, err := sb.stagedSectorFile(sectorId)
if err != nil {
return 0, err
return PublicPieceInfo{}, err
}
return sectorID, werr()
_, _, commP, err := sectorbuilder.StandaloneWriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
if err != nil {
return PublicPieceInfo{}, err
}
if err := stagedFile.Close(); err != nil {
return PublicPieceInfo{}, err
}
if err := f.Close(); err != nil {
return PublicPieceInfo{}, err
}
return PublicPieceInfo{
Size: pieceSize,
CommP: commP,
}, werr()
}
// TODO: should *really really* return an io.ReadCloser
func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, error) {
ret := sb.rlimit()
defer ret()
return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey)
}
func (sb *SectorBuilder) SealSector(sectorID uint64, ticket SealTicket) (SealedSectorMetadata, error) {
return sectorbuilder.SealSector(sb.handle, sectorID, ticket)
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
ret := sb.rlimit()
defer ret()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, err
}
sealedPath, err := sb.sealedSectorPath(sectorID)
if err != nil {
return RawSealPreCommitOutput{}, err
}
var sum uint64
for _, piece := range pieces {
sum += piece.Size
}
ussize := UserBytesForSectorSize(sb.ssize)
if sum != ussize {
return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
}
stagedPath := sb.stagedSectorPath(sectorID)
rspco, err := sectorbuilder.StandaloneSealPreCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
stagedPath,
sealedPath,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
pieces,
)
if err != nil {
return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err)
}
return rspco, nil
}
func (sb *SectorBuilder) ResumeSealSector(sectorID uint64) (SealedSectorMetadata, error) {
return sectorbuilder.ResumeSealSector(sb.handle, sectorID)
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, pieceKeys []string, rspco RawSealPreCommitOutput) (proof []byte, err error) {
ret := sb.rlimit()
defer ret()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
proof, err = sectorbuilder.StandaloneSealCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
sectorID,
addressToProverID(sb.Miner),
ticket.TicketBytes,
seed.TicketBytes,
pieces,
rspco,
)
if err != nil {
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
}
pmeta := make([]sectorbuilder.PieceMetadata, len(pieces))
for i, piece := range pieces {
pmeta[i] = sectorbuilder.PieceMetadata{
Key: pieceKeys[i],
Size: piece.Size,
CommP: piece.CommP,
}
}
sealedPath, err := sb.sealedSectorPath(sectorID)
if err != nil {
return nil, err
}
err = sectorbuilder.ImportSealedSector(
sb.handle,
sectorID,
cacheDir,
sealedPath,
ticket,
seed,
rspco.CommR,
rspco.CommD,
rspco.CommC,
rspco.CommRLast,
proof,
pmeta,
)
if err != nil {
return nil, xerrors.Errorf("ImportSealedSector: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) {
@ -123,24 +337,21 @@ func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed
return sectorbuilder.GeneratePoSt(sb.handle, sectorInfo, challengeSeed, faults)
}
func (sb *SectorBuilder) SectorSize() uint64 {
return sb.ssize
}
var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, sectorID uint64, proof []byte) (bool, error) {
var commRa, commDa, ticketa [32]byte
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
var commRa, commDa, ticketa, seeda [32]byte
copy(commRa[:], commR)
copy(commDa[:], commD)
copy(ticketa[:], ticket)
copy(seeda[:], seed)
proverIDa := addressToProverID(proverID)
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, sectorID, proof)
}
func VerifyPieceInclusionProof(sectorSize uint64, pieceSize uint64, commP []byte, commD []byte, proof []byte) (bool, error) {
var commPa, commDa [32]byte
copy(commPa[:], commP)
copy(commDa[:], commD)
return sectorbuilder.VerifyPieceInclusionProof(sectorSize, pieceSize, commPa, commDa, proof)
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
}
func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo {
@ -165,36 +376,6 @@ func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]
return commP, werr()
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var wait sync.Mutex
var werr error
wait.Lock()
go func() {
defer wait.Unlock()
_, werr = io.CopyN(w, r, n)
err := w.Close()
if werr == nil {
werr = err
}
}()
return f, func() error {
wait.Lock()
return werr
}, nil
func GenerateDataCommitment(ssize uint64, pieces []PublicPieceInfo) ([CommLen]byte, error) {
return sectorbuilder.GenerateDataCommitment(ssize, pieces)
}

121
lib/sectorbuilder/sectorbuilder_test.go
View File

@ -1,64 +1,137 @@
package sectorbuilder_test
import (
"context"
"io"
"io/ioutil"
"math/rand"
"os"
"testing"
"github.com/ipfs/go-datastore"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/storage/sector"
)
const sectorSize = 1024
func TestSealAndVerify(t *testing.T) {
t.Skip("this is slow")
//t.Skip("this is slow")
os.Setenv("BELLMAN_NO_GPU", "1")
build.SectorSizes = []uint64{sectorSize}
if err := build.GetParams(true); err != nil {
t.Fatal(err)
t.Fatalf("%+v", err)
}
sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize)
sb, cleanup, err := sectorbuilder.TempSectorbuilder(sectorSize, datastore.NewMapDatastore())
if err != nil {
t.Fatal(err)
t.Fatalf("%+v", err)
}
defer cleanup()
// TODO: Consider fixing
store := sector.NewStore(sb, datastore.NewMapDatastore(), func(ctx context.Context) (*sectorbuilder.SealTicket, error) {
return &sectorbuilder.SealTicket{
BlockHeight: 5,
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
}, nil
})
store.Service()
dlen := sectorbuilder.UserBytesForSectorSize(sectorSize)
sid, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
r := io.LimitReader(rand.New(rand.NewSource(42)), int64(dlen))
sid, err := store.AddPiece("foo", dlen, r)
ppi, err := sb.AddPiece(dlen, sid, r, []uint64{})
if err != nil {
t.Fatal(err)
t.Fatalf("%+v", err)
}
if err := store.SealSector(context.TODO(), sid); err != nil {
t.Fatal(err)
ticket := sectorbuilder.SealTicket{
BlockHeight: 5,
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
}
ssinfo := <-store.Incoming()
ok, err := sectorbuilder.VerifySeal(sectorSize, ssinfo.CommR[:], ssinfo.CommD[:], sb.Miner, ssinfo.Ticket.TicketBytes[:], ssinfo.SectorID, ssinfo.Proof)
pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{ppi})
if err != nil {
t.Fatal(err)
t.Fatalf("%+v", err)
}
seed := sectorbuilder.SealSeed{
BlockHeight: 15,
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
}
proof, err := sb.SealCommit(sid, ticket, seed, []sectorbuilder.PublicPieceInfo{ppi}, []string{"foo"}, pco)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], pco.CommD[:], sb.Miner, ticket.TicketBytes[:], seed.TicketBytes[:], sid, proof)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("proof failed to validate")
}
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
ssi := sectorbuilder.NewSortedSectorInfo([]sectorbuilder.SectorInfo{{
SectorID: sid,
CommR: pco.CommR,
}})
postProof, err := sb.GeneratePoSt(ssi, cSeed, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
ok, err = sectorbuilder.VerifyPost(sb.SectorSize(), ssi, cSeed, postProof, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("bad post")
}
}
func TestAcquireID(t *testing.T) {
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire := func(expect uint64) {
id, err := sb.AcquireSectorId()
require.NoError(t, err)
assert.Equal(t, expect, id)
}
assertAcquire(1)
assertAcquire(2)
assertAcquire(3)
sb.Destroy()
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
assertAcquire(4)
assertAcquire(5)
assertAcquire(6)
sb.Destroy()
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}

161
lib/statestore/store.go Normal file
View File

@ -0,0 +1,161 @@
package statestore
import (
"bytes"
"fmt"
"reflect"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/lib/cborutil"
)
type StateStore struct {
ds datastore.Datastore
}
func New(ds datastore.Datastore) *StateStore {
return &StateStore{ds: ds}
}
func toKey(k interface{}) datastore.Key {
switch t := k.(type) {
case uint64:
return datastore.NewKey(fmt.Sprint(t))
case fmt.Stringer:
return datastore.NewKey(t.String())
default:
panic("unexpected key type")
}
}
func (st *StateStore) Begin(i interface{}, state interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if has {
return xerrors.Errorf("already tracking state for %v", i)
}
b, err := cborutil.Dump(state)
if err != nil {
return err
}
return st.ds.Put(k, b)
}
func (st *StateStore) End(i interface{}) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
return st.ds.Delete(k)
}
func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
rmut := reflect.ValueOf(mutator)
return func(in []byte) ([]byte, error) {
state := reflect.New(rmut.Type().In(0).Elem())
err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
if err != nil {
return nil, err
}
out := rmut.Call([]reflect.Value{state})
if err := out[0].Interface(); err != nil {
return nil, err.(error)
}
return cborutil.Dump(state.Interface())
}
}
// mutator func(*T) error
func (st *StateStore) Mutate(i interface{}, mutator interface{}) error {
return st.mutate(i, cborMutator(mutator))
}
func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error {
k := toKey(i)
has, err := st.ds.Has(k)
if err != nil {
return err
}
if !has {
return xerrors.Errorf("No state for %s", i)
}
cur, err := st.ds.Get(k)
if err != nil {
return err
}
mutated, err := mutator(cur)
if err != nil {
return err
}
return st.ds.Put(k, mutated)
}
func (st *StateStore) Has(i interface{}) (bool, error) {
return st.ds.Has(toKey(i))
}
func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
k := toKey(i)
val, err := st.ds.Get(k)
if err != nil {
if xerrors.Is(err, datastore.ErrNotFound) {
return xerrors.Errorf("No state for %s: %w", i, err)
}
return err
}
return out.UnmarshalCBOR(bytes.NewReader(val))
}
// out: *[]T
func (st *StateStore) List(out interface{}) error {
res, err := st.ds.Query(query.Query{})
if err != nil {
return err
}
defer res.Close()
outT := reflect.TypeOf(out).Elem().Elem()
rout := reflect.ValueOf(out)
for {
res, ok := res.NextSync()
if !ok {
break
}
if res.Error != nil {
return res.Error
}
elem := reflect.New(outT)
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
if err != nil {
return err
}
rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
}
return nil
}

38
lib/statestore/store_test.go Normal file
View File

@ -0,0 +1,38 @@
package statestore
import (
"testing"
"github.com/ipfs/go-datastore"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborutil"
)
func TestList(t *testing.T) {
ds := datastore.NewMapDatastore()
e, err := cborutil.Dump(types.NewInt(7))
if err != nil {
t.Fatal(err)
}
if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
t.Fatal(err)
}
st := &StateStore{ds: ds}
var out []types.BigInt
if err := st.List(&out); err != nil {
t.Fatal(err)
}
if len(out) != 1 {
t.Fatal("wrong len")
}
if out[0].Int64() != 7 {
t.Fatal("wrong data")
}
}

4
lotuspond/front/src/Client.js
View File

@ -21,7 +21,7 @@ class Client extends React.Component {
this.state = {
miners: ["t0101"],
ask: {Price: "500000000"},
ask: {Price: "1000000000"}, // 2x min default ask to account for bin packing (could also do the math correctly below, but..)
kbs: 1,
blocks: 12,
@ -52,7 +52,7 @@ class Client extends React.Component {
update = (name) => (e) => this.setState({ [name]: e.target.value });
makeDeal = async () => {
let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30)
let perBlk = this.state.ask.Price * this.state.kbs * 1000 / (1 << 30) * 2
let file = await this.props.pondClient.call('Pond.CreateRandomFile', [this.state.kbs * 1000]) // 1024 won't fit in 1k blocks :(
let cid = await this.props.client.call('Filecoin.ClientImport', [file])

15
lotuspond/front/src/StorageNode.js
View File

@ -8,13 +8,14 @@ const stateConnecting = 'connecting'
const stateGettingToken = 'getting-token'
let sealCodes = [
"Unknown",
"Pending",
"Failed",
"Sealing",
"Sealed",
"Paused",
"ReadyForSealing",
"Undefined",
"Empty",
"Packing",
"Unsealed",
"PreCommitting",
"PreCommitted",
"Committing",
"Proving",
]
class StorageNode extends React.Component {

9
lotuspond/front/src/chain/methods.js
View File

@ -34,8 +34,9 @@ export default {
[code.miner]: [
"Send",
"Constructor",
"CommitSector",
"SubmitPost",
"PreCommitSector",
"ProveCommitSector",
"SubmitPoSt",
"SlashStorageFault",
"GetCurrentProvingSet",
"ArbitrateDeal",
@ -49,8 +50,8 @@ export default {
"ChangeWorker",
"IsSlashed",
"IsLate",
"PaymentVerifyInclusion",
"PaymentVerifySector",
"DeclareFaults",
"SlashConsensusFault",
],
[code.multisig]: [
"Send",

6
lotuspond/spawn.go
View File

@ -49,7 +49,7 @@ func (api *api) Spawn() (nodeInfo, error) {
cmd := exec.Command("./lotus", "daemon", "--bootstrap=false", genParam, "--api", fmt.Sprintf("%d", 2500+id))
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw)
cmd.Env = []string{"LOTUS_PATH=" + dir}
cmd.Env = append(os.Environ(), "LOTUS_PATH="+dir)
if err := cmd.Start(); err != nil {
return nodeInfo{}, err
}
@ -112,7 +112,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) {
cmd := exec.Command("./lotus-storage-miner", initArgs...)
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile)
cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo}
cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo)
if err := cmd.Run(); err != nil {
return nodeInfo{}, err
}
@ -124,7 +124,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) {
cmd = exec.Command("./lotus-storage-miner", "run", "--api", fmt.Sprintf("%d", 2500+id))
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw)
cmd.Env = []string{"LOTUS_STORAGE_PATH=" + dir, "LOTUS_PATH=" + fullNodeRepo}
cmd.Env = append(os.Environ(), "LOTUS_STORAGE_PATH="+dir, "LOTUS_PATH="+fullNodeRepo)
if err := cmd.Start(); err != nil {
return nodeInfo{}, err
}

2
miner/miner.go
View File

@ -230,7 +230,7 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error)
}
func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*types.BlockMsg, error) {
log.Infow("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
ticket, err := m.scratchTicket(ctx, base)
if err != nil {
return nil, errors.Wrap(err, "scratching ticket failed")

11
node/builder.go
View File

@ -20,6 +20,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/metrics"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
@ -41,8 +42,6 @@ import (
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/commitment"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -223,17 +222,16 @@ func Online() Option {
Override(new(*paych.Store), paych.NewStore),
Override(new(*paych.Manager), paych.NewManager),
Override(new(*market.FundMgr), market.NewFundMgr),
Override(new(*miner.Miner), miner.NewMiner),
),
// Storage miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.RepoStorageMiner },
Override(new(*sectorbuilder.SectorBuilder), sectorbuilder.New),
Override(new(*sector.Store), sector.NewStore),
Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder),
Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
Override(new(*commitment.Tracker), commitment.NewTracker),
Override(new(sector.TicketFn), modules.SealTicketGen),
Override(new(storage.TicketFn), modules.SealTicketGen),
Override(new(*storage.Miner), modules.StorageMiner),
Override(new(dtypes.StagingDAG), modules.StagingDAG),
@ -242,7 +240,6 @@ func Online() Option {
Override(new(*deals.Provider), deals.NewProvider),
Override(HandleRetrievalKey, modules.HandleRetrieval),
Override(HandleDealsKey, modules.HandleDeals),
Override(RunSectorServiceKey, modules.RunSectorService),
Override(RegisterMinerKey, modules.RegisterMiner),
),
)

6
node/hello/hello.go
View File

@ -16,7 +16,7 @@ import (
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/peermgr"
)
@ -66,7 +66,7 @@ func (hs *Service) HandleStream(s inet.Stream) {
defer s.Close()
var hmsg Message
if err := cborrpc.ReadCborRPC(s, &hmsg); err != nil {
if err := cborutil.ReadCborRPC(s, &hmsg); err != nil {
log.Infow("failed to read hello message", "error", err)
return
}
@ -120,7 +120,7 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error {
fmt.Println("SENDING HELLO MESSAGE: ", hts.Cids(), hts.Height())
fmt.Println("hello message genesis: ", gen.Cid())
if err := cborrpc.WriteCborRPC(s, hmsg); err != nil {
if err := cborutil.WriteCborRPC(s, hmsg); err != nil {
return err
}

51
node/impl/client/client.go
View File

@ -24,7 +24,6 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/store"
@ -58,23 +57,17 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
// TODO: make this a param
self, err := a.WalletDefaultAddress(ctx)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed to get default address: %w", err)
}
// get miner peerID
msg := &types.Message{
To: miner,
From: miner,
Method: actors.MAMethods.GetPeerID,
pid, err := a.StateMinerPeerID(ctx, miner, nil)
if err != nil {
return nil, xerrors.Errorf("failed getting peer ID: %w", err)
}
r, err := a.StateCall(ctx, msg, nil)
mw, err := a.StateMinerWorker(ctx, miner, nil)
if err != nil {
return nil, err
}
pid, err := peer.IDFromBytes(r.Return)
if err != nil {
return nil, err
return nil, xerrors.Errorf("failed getting miner worker: %w", err)
}
proposal := deals.ClientDealProposal{
@ -82,14 +75,18 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.A
PricePerEpoch: epochPrice,
ProposalExpiration: math.MaxUint64, // TODO: set something reasonable
Duration: blocksDuration,
ProviderAddress: miner,
Client: self,
ProviderAddress: miner,
MinerWorker: mw,
MinerID: pid,
}
c, err := a.DealClient.Start(ctx, proposal)
// TODO: send updated voucher with PaymentVerifySector for cheaper validation (validate the sector the miner sent us first!)
return &c, err
if err != nil {
return nil, xerrors.Errorf("failed to start deal: %w", err)
}
return &c, nil
}
func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
@ -116,6 +113,22 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
return out, nil
}
func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) {
v, err := a.DealClient.GetDeal(d)
if err != nil {
return nil, err
}
return &api.DealInfo{
ProposalCid: v.ProposalCid,
State: v.State,
Provider: v.Proposal.Provider,
PieceRef: v.Proposal.PieceRef,
Size: v.Proposal.PieceSize,
PricePerEpoch: v.Proposal.StoragePricePerEpoch,
Duration: v.Proposal.Duration,
}, nil
}
func (a *API) ClientHasLocal(ctx context.Context, root cid.Cid) (bool, error) {
// TODO: check if we have the ENTIRE dag
@ -178,7 +191,11 @@ func (a *API) ClientImport(ctx context.Context, path string) (cid.Cid, error) {
return cid.Undef, err
}
return nd.Cid(), bufferedDS.Commit()
if err := bufferedDS.Commit(); err != nil {
return cid.Undef, err
}
return nd.Cid(), nil
}
func (a *API) ClientImportLocal(ctx context.Context, f io.Reader) (cid.Cid, error) {

8
node/impl/full.go
View File

@ -3,11 +3,12 @@ package impl
import (
"context"
"github.com/filecoin-project/lotus/node/impl/client"
"github.com/filecoin-project/lotus/node/impl/paych"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/node/impl/client"
"github.com/filecoin-project/lotus/node/impl/market"
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/miner"
@ -21,6 +22,7 @@ type FullNodeAPI struct {
full.ChainAPI
client.API
full.MpoolAPI
market.MarketAPI
paych.PaychAPI
full.StateAPI
full.WalletAPI

11
node/impl/full/state.go
View File

@ -3,9 +3,10 @@ package full
import (
"bytes"
"context"
"github.com/filecoin-project/go-amt-ipld"
"strconv"
"github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-hamt-ipld"
"github.com/libp2p/go-libp2p-core/peer"
@ -37,11 +38,11 @@ type StateAPI struct {
Chain *store.ChainStore
}
func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) {
func (a *StateAPI) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
return stmgr.GetMinerSectorSet(ctx, a.StateManager, ts, addr)
}
func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.SectorInfo, error) {
func (a *StateAPI) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
return stmgr.GetMinerProvingSet(ctx, a.StateManager, ts, addr)
}
@ -279,3 +280,7 @@ func (a *StateAPI) StateMarketDeals(ctx context.Context, ts *types.TipSet) (map[
}
return out, nil
}
func (a *StateAPI) StateMarketStorageDeal(ctx context.Context, dealId uint64, ts *types.TipSet) (*actors.OnChainDeal, error) {
return stmgr.GetStorageDeal(ctx, a.StateManager, dealId, ts)
}

21
node/impl/market/market.go Normal file
View File

@ -0,0 +1,21 @@
package market
import (
"context"
"go.uber.org/fx"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/types"
)
type MarketAPI struct {
fx.In
FMgr *market.FundMgr
}
func (a *MarketAPI) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error {
return a.FMgr.EnsureAvailable(ctx, addr, amt)
}

82
node/impl/storminer.go
View File

@ -2,31 +2,32 @@ package impl
import (
"context"
"fmt"
"io"
"math/rand"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/commitment"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/storage/sectorblocks"
"golang.org/x/xerrors"
)
type StorageMinerAPI struct {
CommonAPI
SectorBuilderConfig *sectorbuilder.SectorBuilderConfig
SectorBuilderConfig *sectorbuilder.Config
SectorBuilder *sectorbuilder.SectorBuilder
Sectors *sector.Store
SectorBlocks *sectorblocks.SectorBlocks
CommitmentTracker *commitment.Tracker
Miner *storage.Miner
Full api.FullNode
}
func (sm *StorageMinerAPI) WorkerStats(context.Context) (api.WorkerStats, error) {
free, reserved, total := sm.SectorBuilder.WorkerStats()
return api.WorkerStats{
Free: free,
Reserved: reserved,
Total: total,
}, nil
}
func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) {
@ -34,37 +35,44 @@ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error
}
func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) error {
ssize, err := sm.Miner.SectorSize(ctx)
if err != nil {
return xerrors.Errorf("failed to get miner sector size: %w", err)
}
go func() {
size := sectorbuilder.UserBytesForSectorSize(ssize)
// TODO: create a deal
name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000))
sectorId, err := sm.Sectors.AddPiece(name, size, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)))
if err != nil {
log.Error(err)
return
}
if err := sm.Sectors.SealSector(ctx, sectorId); err != nil {
log.Error(err)
return
}
}()
return err
return sm.Miner.StoreGarbageData()
}
func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) {
return sm.SectorBuilder.SealStatus(sid)
func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) {
info, err := sm.Miner.GetSectorInfo(sid)
if err != nil {
return api.SectorInfo{}, err
}
deals := make([]uint64, len(info.Pieces))
for i, piece := range info.Pieces {
deals[i] = piece.DealID
}
return api.SectorInfo{
SectorID: sid,
State: info.State,
CommD: info.CommD,
CommR: info.CommR,
Proof: info.Proof,
Deals: deals,
Ticket: info.Ticket.SB(),
Seed: info.Seed.SB(),
}, nil
}
// List all staged sectors
func (sm *StorageMinerAPI) SectorsList(context.Context) ([]uint64, error) {
return sm.SectorBuilder.GetAllStagedSectors()
sectors, err := sm.Miner.ListSectors()
if err != nil {
return nil, err
}
out := make([]uint64, len(sectors))
for i, sector := range sectors {
out[i] = sector.SectorID
}
return out, nil
}
func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) {
@ -83,8 +91,4 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed
return out, nil
}
func (sm *StorageMinerAPI) CommitmentsList(ctx context.Context) ([]api.SectorCommitment, error) {
return sm.CommitmentTracker.List()
}
var _ api.StorageMiner = &StorageMinerAPI{}

14
node/modules/services.go
View File

@ -15,7 +15,6 @@ import (
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/peermgr"
"github.com/filecoin-project/lotus/retrieval/discovery"
"github.com/filecoin-project/lotus/storage/sector"
)
func RunHello(mctx helpers.MetricsCtx, lc fx.Lifecycle, h host.Host, svc *hello.Service) {
@ -79,19 +78,6 @@ func RunDealClient(mctx helpers.MetricsCtx, lc fx.Lifecycle, c *deals.Client) {
})
}
func RunSectorService(lc fx.Lifecycle, secst *sector.Store) {
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
secst.Service()
return nil
},
OnStop: func(context.Context) error {
secst.Stop()
return nil
},
})
}
func RetrievalResolver(l *discovery.Local) discovery.PeerResolver {
return discovery.Multi(l)
}

46
node/modules/storageminer.go
View File

@ -3,6 +3,7 @@ package modules
import (
"context"
"fmt"
"math"
"path/filepath"
"github.com/ipfs/go-bitswap"
@ -27,8 +28,6 @@ import (
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/commitment"
"github.com/filecoin-project/lotus/storage/sector"
)
func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
@ -40,8 +39,8 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
return address.NewFromBytes(maddrb)
}
func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) {
return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.SectorBuilderConfig, error) {
func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) {
return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
@ -57,13 +56,21 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod
return nil, err
}
if threads > math.MaxUint8 {
return nil, xerrors.Errorf("too many sectorbuilder threads specified: %d, max allowed: %d", threads, math.MaxUint8)
}
cache := filepath.Join(sp, "cache")
metadata := filepath.Join(sp, "meta")
sealed := filepath.Join(sp, "sealed")
staging := filepath.Join(sp, "staging")
sb := &sectorbuilder.SectorBuilderConfig{
Miner: minerAddr,
SectorSize: ssize,
sb := &sectorbuilder.Config{
Miner: minerAddr,
SectorSize: ssize,
WorkerThreads: uint8(threads),
CacheDir: cache,
MetadataDir: metadata,
SealedDir: sealed,
StagedDir: staging,
@ -73,13 +80,13 @@ func SectorBuilderConfig(storagePath string) func(dtypes.MetadataDS, api.FullNod
}
}
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, secst *sector.Store, commt *commitment.Tracker) (*storage.Miner, error) {
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
sm, err := storage.NewMiner(api, maddr, h, ds, secst, commt)
sm, err := storage.NewMiner(api, maddr, h, ds, sb, tktFn)
if err != nil {
return nil, err
}
@ -90,6 +97,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h
OnStart: func(context.Context) error {
return sm.Run(ctx)
},
OnStop: sm.Stop,
})
return sm, nil
@ -168,7 +176,23 @@ func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) erro
return nil
}
func SealTicketGen(api api.FullNode) sector.TicketFn {
func SectorBuilder(lc fx.Lifecycle, cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) {
sb, err := sectorbuilder.New(cfg, ds)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
sb.Destroy()
return nil
},
})
return sb, nil
}
func SealTicketGen(api api.FullNode) storage.TicketFn {
return func(ctx context.Context) (*sectorbuilder.SealTicket, error) {
ts, err := api.ChainHead(ctx)
if err != nil {
@ -186,7 +210,7 @@ func SealTicketGen(api api.FullNode) sector.TicketFn {
}
return &sectorbuilder.SealTicket{
BlockHeight: ts.Height() - build.SealRandomnessLookback,
BlockHeight: ts.Height(),
TicketBytes: tkt,
}, nil
}

4
node/modules/testing/genesis.go
View File

@ -25,7 +25,7 @@ import (
var glog = logging.Logger("genesis")
func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
@ -38,7 +38,7 @@ func MakeGenesisMem(out io.Writer) func(bs dtypes.ChainBlockstore, w *wallet.Wal
gmc := &gen.GenMinerCfg{
Owners: []address.Address{w},
Workers: []address.Address{w},
PeerIDs: []peer.ID{"peerID 1"},
PeerIDs: []peer.ID{minerPid},
}
alloc := map[address.Address]types.BigInt{
w: types.FromFil(10000),

23
node/node_test.go
View File

@ -31,15 +31,12 @@ import (
"github.com/filecoin-project/lotus/node/repo"
)
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, tnd test.TestNode) test.TestStorageNode {
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
r := repo.NewMemory(nil)
lr, err := r.Lock(repo.RepoStorageMiner)
require.NoError(t, err)
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
ks, err := lr.KeyStore()
require.NoError(t, err)
@ -93,7 +90,9 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
node.Repo(r),
node.Test(),
node.Override(new(*sectorbuilder.SectorBuilderConfig), modules.SectorBuilderConfig(secbpath)),
node.MockHost(mn),
node.Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(secbpath, 2)),
node.Override(new(api.FullNode), tnd),
)
require.NoError(t, err)
@ -115,12 +114,18 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
fulls := make([]test.TestNode, nFull)
storers := make([]test.TestStorageNode, len(storage))
pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
require.NoError(t, err)
minerPid, err := peer.IDFromPrivateKey(pk)
require.NoError(t, err)
var genbuf bytes.Buffer
for i := 0; i < nFull; i++ {
var genesis node.Option
if i == 0 {
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf))
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid))
} else {
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
@ -171,7 +176,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
genMiner, err := address.NewFromString("t0101")
require.NoError(t, err)
storers[i] = testStorageNode(ctx, t, wa, genMiner, f)
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
}
if err := mn.LinkAll(); err != nil {
@ -221,3 +226,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
func TestAPIRPC(t *testing.T) {
test.TestApis(t, rpcBuilder)
}
func TestAPIDealFlow(t *testing.T) {
test.TestDealFlow(t, builder)
}

3
node/options.go
View File

@ -1,8 +1,9 @@
package node
import (
"go.uber.org/fx"
"reflect"
"go.uber.org/fx"
)
// Option is a functional option which can be used with the New function to

2
paych/store.go
View File

@ -14,7 +14,7 @@ import (
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
cborrpc "github.com/filecoin-project/lotus/lib/cborrpc"
cborrpc "github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)

3
peermgr/peermgr.go
View File

@ -2,10 +2,11 @@ package peermgr
import (
"context"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"sync"
"time"
"github.com/filecoin-project/lotus/node/modules/dtypes"
host "github.com/libp2p/go-libp2p-core/host"
net "github.com/libp2p/go-libp2p-core/network"
peer "github.com/libp2p/go-libp2p-core/peer"

10
retrieval/client.go
View File

@ -17,7 +17,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
payapi "github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/paych"
"github.com/filecoin-project/lotus/retrieval/discovery"
@ -44,7 +44,7 @@ func (c *Client) Query(ctx context.Context, p discovery.RetrievalPeer, data cid.
}
defer s.Close()
err = cborrpc.WriteCborRPC(s, &Query{
err = cborutil.WriteCborRPC(s, &Query{
Piece: data,
})
if err != nil {
@ -172,12 +172,12 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out
},
}
if err := cborrpc.WriteCborRPC(cst.stream, deal); err != nil {
if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil {
return err
}
var resp DealResponse
if err := cborrpc.ReadCborRPC(cst.peeker, &resp); err != nil {
if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil {
log.Error(err)
return err
}
@ -209,7 +209,7 @@ func (cst *clientStream) fetchBlocks(toFetch uint64, out io.Writer) error {
log.Infof("block %d of %d", i+1, blocksToFetch)
var block Block
if err := cborrpc.ReadCborRPC(cst.peeker, &block); err != nil {
if err := cborutil.ReadCborRPC(cst.peeker, &block); err != nil {
return xerrors.Errorf("reading fetchBlock response: %w", err)
}

14
retrieval/miner.go
View File

@ -15,7 +15,7 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborrpc"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -42,7 +42,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner {
func writeErr(stream network.Stream, err error) {
log.Errorf("Retrieval deal error: %s", err)
_ = cborrpc.WriteCborRPC(stream, &DealResponse{
_ = cborutil.WriteCborRPC(stream, &DealResponse{
Status: Error,
Message: err.Error(),
})
@ -52,7 +52,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) {
defer stream.Close()
var query Query
if err := cborrpc.ReadCborRPC(stream, &query); err != nil {
if err := cborutil.ReadCborRPC(stream, &query); err != nil {
writeErr(stream, err)
return
}
@ -74,7 +74,7 @@ func (m *Miner) HandleQueryStream(stream network.Stream) {
answer.Size = uint64(size) // TODO: verify on intermediate
}
if err := cborrpc.WriteCborRPC(stream, answer); err != nil {
if err := cborutil.WriteCborRPC(stream, answer); err != nil {
log.Errorf("Retrieval query: WriteCborRPC: %s", err)
return
}
@ -114,7 +114,7 @@ func (m *Miner) HandleDealStream(stream network.Stream) {
func (hnd *handlerDeal) handleNext() (bool, error) {
var deal DealProposal
if err := cborrpc.ReadCborRPC(hnd.stream, &deal); err != nil {
if err := cborutil.ReadCborRPC(hnd.stream, &deal); err != nil {
if err == io.EOF { // client sent all deals
err = nil
}
@ -203,7 +203,7 @@ func (hnd *handlerDeal) accept(deal DealProposal) error {
resp := &DealResponse{
Status: Accepted,
}
if err := cborrpc.WriteCborRPC(hnd.stream, resp); err != nil {
if err := cborutil.WriteCborRPC(hnd.stream, resp); err != nil {
log.Errorf("Retrieval query: Write Accepted resp: %s", err)
return err
}
@ -231,7 +231,7 @@ func (hnd *handlerDeal) accept(deal DealProposal) error {
Data: nd.RawData(),
}
if err := cborrpc.WriteCborRPC(hnd.stream, block); err != nil {
if err := cborutil.WriteCborRPC(hnd.stream, block); err != nil {
return err
}

574
storage/cbor_gen.go Normal file
View File

@ -0,0 +1,574 @@
package storage
import (
"fmt"
"io"
cbg "github.com/whyrusleeping/cbor-gen"
xerrors "golang.org/x/xerrors"
)
/* This file was generated by github.com/whyrusleeping/cbor-gen */
var _ = xerrors.Errorf
func (t *SealTicket) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.t.BlockHeight (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil {
return err
}
// t.t.TicketBytes ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil {
return err
}
if _, err := w.Write(t.TicketBytes); err != nil {
return err
}
return nil
}
func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.BlockHeight (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.BlockHeight = uint64(extra)
// t.t.TicketBytes ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.TicketBytes: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.TicketBytes = make([]byte, extra)
if _, err := io.ReadFull(br, t.TicketBytes); err != nil {
return err
}
return nil
}
func (t *SealSeed) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{130}); err != nil {
return err
}
// t.t.BlockHeight (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.BlockHeight))); err != nil {
return err
}
// t.t.TicketBytes ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketBytes)))); err != nil {
return err
}
if _, err := w.Write(t.TicketBytes); err != nil {
return err
}
return nil
}
func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.BlockHeight (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.BlockHeight = uint64(extra)
// t.t.TicketBytes ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.TicketBytes: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.TicketBytes = make([]byte, extra)
if _, err := io.ReadFull(br, t.TicketBytes); err != nil {
return err
}
return nil
}
func (t *Piece) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{132}); err != nil {
return err
}
// t.t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
// t.t.Ref (string) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Ref)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Ref)); err != nil {
return err
}
// t.t.Size (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
return err
}
// t.t.CommP ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommP)))); err != nil {
return err
}
if _, err := w.Write(t.CommP); err != nil {
return err
}
return nil
}
func (t *Piece) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 4 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
// t.t.Ref (string) (string)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Ref = string(sval)
}
// t.t.Size (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.Size = uint64(extra)
// t.t.CommP ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommP: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommP = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommP); err != nil {
return err
}
return nil
}
func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{140}); err != nil {
return err
}
// t.t.State (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
return err
}
// t.t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
// t.t.Pieces ([]storage.Piece) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil {
return err
}
for _, v := range t.Pieces {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.t.CommC ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommC)))); err != nil {
return err
}
if _, err := w.Write(t.CommC); err != nil {
return err
}
// t.t.CommD ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommD)))); err != nil {
return err
}
if _, err := w.Write(t.CommD); err != nil {
return err
}
// t.t.CommR ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommR)))); err != nil {
return err
}
if _, err := w.Write(t.CommR); err != nil {
return err
}
// t.t.CommRLast ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.CommRLast)))); err != nil {
return err
}
if _, err := w.Write(t.CommRLast); err != nil {
return err
}
// t.t.Proof ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil {
return err
}
if _, err := w.Write(t.Proof); err != nil {
return err
}
// t.t.Ticket (storage.SealTicket) (struct)
if err := t.Ticket.MarshalCBOR(w); err != nil {
return err
}
// t.t.PreCommitMessage (cid.Cid) (struct)
if t.PreCommitMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.PreCommitMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.PreCommitMessage: %w", err)
}
}
// t.t.Seed (storage.SealSeed) (struct)
if err := t.Seed.MarshalCBOR(w); err != nil {
return err
}
// t.t.CommitMessage (cid.Cid) (struct)
if t.CommitMessage == nil {
if _, err := w.Write(cbg.CborNull); err != nil {
return err
}
} else {
if err := cbg.WriteCid(w, *t.CommitMessage); err != nil {
return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err)
}
}
return nil
}
func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 12 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.State (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.State = uint64(extra)
// t.t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
// t.t.Pieces ([]storage.Piece) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Pieces: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Pieces = make([]Piece, extra)
}
for i := 0; i < int(extra); i++ {
var v Piece
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Pieces[i] = v
}
// t.t.CommC ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommC: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommC = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommC); err != nil {
return err
}
// t.t.CommD ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommD: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommD = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommD); err != nil {
return err
}
// t.t.CommR ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommR: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommR = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommR); err != nil {
return err
}
// t.t.CommRLast ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.CommRLast: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.CommRLast = make([]byte, extra)
if _, err := io.ReadFull(br, t.CommRLast); err != nil {
return err
}
// t.t.Proof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Proof: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Proof = make([]byte, extra)
if _, err := io.ReadFull(br, t.Proof); err != nil {
return err
}
// t.t.Ticket (storage.SealTicket) (struct)
{
if err := t.Ticket.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.PreCommitMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err)
}
t.PreCommitMessage = &c
}
}
// t.t.Seed (storage.SealSeed) (struct)
{
if err := t.Seed.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.CommitMessage (cid.Cid) (struct)
{
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
c, err := cbg.ReadCid(br)
if err != nil {
return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err)
}
t.CommitMessage = &c
}
}
return nil
}

198
storage/commitment/tracker.go
View File

@ -1,198 +0,0 @@
package commitment
import (
"context"
"fmt"
"strconv"
"strings"
"sync"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
dsq "github.com/ipfs/go-datastore/query"
)
var log = logging.Logger("commitment")
func init() {
cbor.RegisterCborType(commitment{})
}
var commitmentDsPrefix = datastore.NewKey("/commitments")
type Tracker struct {
commitments datastore.Datastore
lk sync.Mutex
waits map[datastore.Key]chan struct{}
}
func NewTracker(ds dtypes.MetadataDS) *Tracker {
return &Tracker{
commitments: namespace.Wrap(ds, commitmentDsPrefix),
waits: map[datastore.Key]chan struct{}{},
}
}
type commitment struct {
DealIDs []uint64
Msg cid.Cid
}
func commitmentKey(miner address.Address, sectorId uint64) datastore.Key {
return commitmentDsPrefix.ChildString(miner.String()).ChildString(fmt.Sprintf("%d", sectorId))
}
func (ct *Tracker) TrackCommitSectorMsg(miner address.Address, sectorId uint64, commitMsg cid.Cid) error {
key := commitmentKey(miner, sectorId)
ct.lk.Lock()
defer ct.lk.Unlock()
tracking, err := ct.commitments.Get(key)
switch err {
case nil:
var comm commitment
if err := cbor.DecodeInto(tracking, &comm); err != nil {
return err
}
if !comm.Msg.Equals(commitMsg) {
log.Errorf("commitment tracking for miner %s, sector %d: already tracking %s, got another commitment message: %s", miner, sectorId, comm.Msg, commitMsg)
}
log.Warnf("commitment.TrackCommitSectorMsg called more than once for miner %s, sector %d, message %s", miner, sectorId, commitMsg)
// we still want to store it
fallthrough // TODO: ideally we'd keep around both (even though we'll
// usually only need the new one)
case datastore.ErrNotFound:
comm := &commitment{Msg: commitMsg}
commB, err := cbor.DumpObject(comm)
if err != nil {
return err
}
if err := ct.commitments.Put(key, commB); err != nil {
return err
}
waits, ok := ct.waits[key]
if ok {
close(waits)
delete(ct.waits, key)
}
return nil
default:
return err
}
}
func (ct *Tracker) WaitCommit(ctx context.Context, miner address.Address, sectorId uint64) (cid.Cid, error) {
key := commitmentKey(miner, sectorId)
ct.lk.Lock()
tracking, err := ct.commitments.Get(key)
if err != datastore.ErrNotFound {
ct.lk.Unlock()
if err != nil {
return cid.Undef, err
}
var comm commitment
if err := cbor.DecodeInto(tracking, &comm); err != nil {
return cid.Undef, err
}
return comm.Msg, nil
}
wait, ok := ct.waits[key]
if !ok {
wait = make(chan struct{})
ct.waits[key] = wait
}
ct.lk.Unlock()
select {
case <-wait:
tracking, err := ct.commitments.Get(key)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to get commitment after waiting: %w", err)
}
var comm commitment
if err := cbor.DecodeInto(tracking, &comm); err != nil {
return cid.Undef, err
}
return comm.Msg, nil
case <-ctx.Done():
return cid.Undef, ctx.Err()
}
}
func (ct *Tracker) List() ([]api.SectorCommitment, error) {
out := make([]api.SectorCommitment, 0)
ct.lk.Lock()
defer ct.lk.Unlock()
res, err := ct.commitments.Query(dsq.Query{})
if err != nil {
return nil, err
}
defer res.Close()
for {
res, ok := res.NextSync()
if !ok {
break
}
if res.Error != nil {
return nil, xerrors.Errorf("iterating commitments: %w", err)
}
parts := strings.Split(res.Key, "/")
if len(parts) != 4 {
return nil, xerrors.Errorf("expected commitment key to be 4 parts, Key %s", res.Key)
}
miner, err := address.NewFromString(parts[2])
if err != nil {
return nil, xerrors.Errorf("parsing miner address: %w", err)
}
sectorID, err := strconv.ParseInt(parts[3], 10, 64)
if err != nil {
return nil, xerrors.Errorf("parsing sector id: %w", err)
}
var comm commitment
if err := cbor.DecodeInto(res.Value, &comm); err != nil {
return nil, xerrors.Errorf("decoding commitment %s (`% X`): %w", res.Key, res.Value, err)
}
out = append(out, api.SectorCommitment{
SectorID: uint64(sectorID),
Miner: miner,
CommitMsg: comm.Msg,
DealIDs: comm.DealIDs,
})
}
return out, nil
}

140
storage/garbage.go Normal file
View File

@ -0,0 +1,140 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"math"
"math/rand"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {
if len(sizes) == 0 {
return nil, nil
}
deals := make([]actors.StorageDeal, len(sizes))
for i, size := range sizes {
commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size)
if err != nil {
return nil, err
}
sdp := actors.StorageDealProposal{
PieceRef: commP[:],
PieceSize: size,
PieceSerialization: actors.SerializationUnixFSv0,
Client: m.worker,
Provider: m.maddr,
ProposalExpiration: math.MaxUint64,
Duration: math.MaxUint64 / 2, // /2 because overflows
StoragePricePerEpoch: types.NewInt(0),
StorageCollateral: types.NewInt(0),
ProposerSignature: nil,
}
if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &sdp); err != nil {
return nil, xerrors.Errorf("signing storage deal failed: ", err)
}
storageDeal := actors.StorageDeal{
Proposal: sdp,
}
if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil {
return nil, xerrors.Errorf("signing storage deal failed: ", err)
}
deals[i] = storageDeal
}
params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{
Deals: deals,
})
if aerr != nil {
return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr)
}
smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{
To: actors.StorageMarketAddress,
From: m.worker,
Value: types.NewInt(0),
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(1000000),
Method: actors.SMAMethods.PublishStorageDeals,
Params: params,
})
if err != nil {
return nil, err
}
r, err := m.api.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return nil, err
}
if r.Receipt.ExitCode != 0 {
log.Error(xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode))
}
var resp actors.PublishStorageDealResponse
if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
return nil, err
}
if len(resp.DealIDs) != len(sizes) {
return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals")
}
out := make([]Piece, len(sizes))
for i, size := range sizes {
name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000))
ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes)
if err != nil {
return nil, err
}
existingPieceSizes = append(existingPieceSizes, size)
out[i] = Piece{
DealID: resp.DealIDs[i],
Ref: name,
Size: ppi.Size,
CommP: ppi.CommP[:],
}
}
return out, nil
}
func (m *Miner) StoreGarbageData() error {
go func() {
ctx := context.TODO() // we can't use the context from command which invokes
// this, as we run everything here async, and it's cancelled when the
// command exits
size := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize())
sid, err := m.sb.AcquireSectorId()
if err != nil {
log.Errorf("%+v", err)
return
}
pieces, err := m.storeGarbage(ctx, sid, []uint64{}, size)
if err != nil {
log.Errorf("%+v", err)
return
}
if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].Ref, pieces[0].ppi()); err != nil {
log.Errorf("%+v", err)
return
}
}()
return nil
}

186
storage/miner.go
View File

@ -6,20 +6,18 @@ import (
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
logging "github.com/ipfs/go-log"
"github.com/libp2p/go-libp2p-core/host"
"github.com/pkg/errors"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/events"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/storage/commitment"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/lib/statestore"
)
var log = logging.Logger("storageminer")
@ -29,20 +27,24 @@ const PoStConfidence = 3
type Miner struct {
api storageMinerApi
events *events.Events
h host.Host
secst *sector.Store
commt *commitment.Tracker
maddr address.Address
maddr address.Address
worker address.Address
h host.Host
ds datastore.Batching
schedLk sync.Mutex
// PoSt
postLk sync.Mutex
schedPost uint64
// Sealing
sb *sectorbuilder.SectorBuilder
sectors *statestore.StateStore
tktFn TicketFn
sectorIncoming chan *SectorInfo
sectorUpdated chan sectorUpdate
stop chan struct{}
stopped chan struct{}
}
type storageMinerApi interface {
@ -53,8 +55,8 @@ type storageMinerApi interface {
StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error)
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerProvingPeriodEnd(context.Context, address.Address, *types.TipSet) (uint64, error)
StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.SectorInfo, error)
StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)
StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error)
StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error)
@ -66,19 +68,26 @@ type storageMinerApi interface {
ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
WalletSign(context.Context, address.Address, []byte) (*types.Signature, error)
WalletBalance(context.Context, address.Address) (types.BigInt, error)
WalletHas(context.Context, address.Address) (bool, error)
}
func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, secst *sector.Store, commt *commitment.Tracker) (*Miner, error) {
func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datastore.Batching, sb *sectorbuilder.SectorBuilder, tktFn TicketFn) (*Miner, error) {
return &Miner{
api: api,
maddr: addr,
h: h,
ds: ds,
secst: secst,
commt: commt,
sb: sb,
tktFn: tktFn,
sectors: statestore.New(namespace.Wrap(ds, datastore.NewKey("/sectors"))),
sectorIncoming: make(chan *SectorInfo),
sectorUpdated: make(chan sectorUpdate),
stop: make(chan struct{}),
stopped: make(chan struct{}),
}, nil
}
@ -89,135 +98,19 @@ func (m *Miner) Run(ctx context.Context) error {
m.events = events.NewEvents(ctx, m.api)
go m.handlePostingSealedSectors(ctx)
go m.beginPosting(ctx)
go m.sectorStateLoop(ctx)
return nil
}
func (m *Miner) commitUntrackedSectors(ctx context.Context) error {
sealed, err := m.secst.Sealed()
if err != nil {
return err
func (m *Miner) Stop(ctx context.Context) error {
close(m.stop)
select {
case <-m.stopped:
return nil
case <-ctx.Done():
return ctx.Err()
}
chainSectors, err := m.api.StateMinerSectors(ctx, m.maddr, nil)
if err != nil {
return err
}
onchain := map[uint64]struct{}{}
for _, chainSector := range chainSectors {
onchain[chainSector.SectorID] = struct{}{}
}
for _, s := range sealed {
if _, ok := onchain[s.SectorID]; ok {
continue
}
log.Warnf("Missing commitment for sector %d, committing sector", s.SectorID)
if err := m.commitSector(ctx, s); err != nil {
log.Error("Committing uncommitted sector failed: ", err)
}
}
return nil
}
func (m *Miner) handlePostingSealedSectors(ctx context.Context) {
incoming := m.secst.Incoming()
defer m.secst.CloseIncoming(incoming)
if err := m.commitUntrackedSectors(ctx); err != nil {
log.Error(err)
}
for {
select {
case sinfo, ok := <-incoming:
if !ok {
// TODO: set some state variable so that this state can be
// visible via some status command
log.Warn("sealed sector channel closed, aborting process")
return
}
if err := m.commitSector(ctx, sinfo); err != nil {
log.Errorf("failed to commit sector: %s", err)
continue
}
case <-ctx.Done():
log.Warn("exiting seal posting routine")
return
}
}
}
func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSealingStatus) error {
log.Info("committing sector")
ssize, err := m.SectorSize(ctx)
if err != nil {
return xerrors.Errorf("failed to check out own sector size: %w", err)
}
ok, err := sectorbuilder.VerifySeal(ssize, sinfo.CommR[:], sinfo.CommD[:], m.maddr, sinfo.Ticket.TicketBytes[:], sinfo.SectorID, sinfo.Proof)
if err != nil {
log.Error("failed to verify seal we just created: ", err)
}
if !ok {
log.Error("seal we just created failed verification")
}
deals, err := m.secst.DealsForCommit(sinfo.SectorID)
if err != nil {
return xerrors.Errorf("getting sector deals failed: %w", err)
}
params := &actors.OnChainSealVerifyInfo{
CommD: sinfo.CommD[:],
CommR: sinfo.CommR[:],
Proof: sinfo.Proof,
Epoch: sinfo.Ticket.BlockHeight,
DealIDs: deals,
SectorNumber: sinfo.SectorID,
}
enc, aerr := actors.SerializeParams(params)
if aerr != nil {
return errors.Wrap(aerr, "could not serialize commit sector parameters")
}
msg := &types.Message{
To: m.maddr,
From: m.worker,
Method: actors.MAMethods.CommitSector,
Params: enc,
Value: types.NewInt(0), // TODO: need to ensure sufficient collateral
GasLimit: types.NewInt(1000000 /* i dont know help */),
GasPrice: types.NewInt(1),
}
smsg, err := m.api.MpoolPushMessage(ctx, msg)
if err != nil {
return errors.Wrap(err, "pushing message to mpool")
}
go func() {
_, err := m.api.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return
}
m.beginPosting(ctx)
}()
if err := m.commt.TrackCommitSectorMsg(m.maddr, sinfo.SectorID, smsg.Cid()); err != nil {
return xerrors.Errorf("tracking sector commitment: %w", err)
}
return nil
}
func (m *Miner) runPreflightChecks(ctx context.Context) error {
@ -240,8 +133,3 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error {
log.Infof("starting up miner %s, worker addr %s", m.maddr, m.worker)
return nil
}
func (m *Miner) SectorSize(ctx context.Context) (uint64, error) {
// TODO: cache this
return m.api.StateMinerSectorSize(ctx, m.maddr, nil)
}

246
storage/post.go
View File

@ -4,12 +4,15 @@ import (
"context"
"time"
"github.com/ipfs/go-cid"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func (m *Miner) beginPosting(ctx context.Context) {
@ -30,10 +33,10 @@ func (m *Miner) beginPosting(ctx context.Context) {
return
}
m.schedLk.Lock()
m.postLk.Lock()
if m.schedPost > 0 {
log.Warnf("PoSts already running %d", m.schedPost)
m.schedLk.Unlock()
m.postLk.Unlock()
return
}
@ -42,10 +45,10 @@ func (m *Miner) beginPosting(ctx context.Context) {
ppe, _ = actors.ProvingPeriodEnd(ppe, ts.Height()+1)
m.schedPost = ppe
m.schedLk.Unlock()
m.postLk.Unlock()
log.Infof("Scheduling post at height %d", ppe-build.PoStChallangeTime)
err = m.events.ChainAt(m.computePost(m.schedPost), func(ts *types.TipSet) error { // Revert
err = m.events.ChainAt(m.computePost(m.schedPost), func(ctx context.Context, ts *types.TipSet) error { // Revert
// TODO: Cancel post
log.Errorf("TODO: Cancel PoSt, re-run")
return nil
@ -71,20 +74,20 @@ func (m *Miner) scheduleNextPost(ppe uint64) {
ppe = headPPE
}
m.schedLk.Lock()
m.postLk.Lock()
if m.schedPost >= ppe {
// this probably can't happen
log.Errorw("PoSt already scheduled", "schedPost", m.schedPost, "ppe", ppe)
m.schedLk.Unlock()
m.postLk.Unlock()
return
}
m.schedPost = ppe
m.schedLk.Unlock()
m.postLk.Unlock()
log.Infow("scheduling PoSt", "post-height", ppe-build.PoStChallangeTime,
"height", ts.Height(), "ppe", ppe, "proving-period", provingPeriod)
err = m.events.ChainAt(m.computePost(ppe), func(ts *types.TipSet) error { // Revert
err = m.events.ChainAt(m.computePost(ppe), func(ctx context.Context, ts *types.TipSet) error { // Revert
// TODO: Cancel post
log.Errorf("TODO: Cancel PoSt, re-run")
return nil
@ -96,9 +99,164 @@ func (m *Miner) scheduleNextPost(ppe uint64) {
}
}
func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) error {
type post struct {
m *Miner
ppe uint64
ts *types.TipSet
// prep
sset []*api.ChainSectorInfo
r []byte
// run
proof []byte
// commit
smsg cid.Cid
}
func (p *post) doPost(ctx context.Context) error {
ctx, span := trace.StartSpan(ctx, "storage.computePost")
defer span.End()
if err := p.preparePost(ctx); err != nil {
return err
}
if err := p.runPost(ctx); err != nil {
return err
}
if err := p.commitPost(ctx); err != nil {
return err
}
if err := p.waitCommit(ctx); err != nil {
return err
}
return nil
}
func (p *post) preparePost(ctx context.Context) error {
ctx, span := trace.StartSpan(ctx, "storage.preparePost")
defer span.End()
sset, err := p.m.api.StateMinerProvingSet(ctx, p.m.maddr, p.ts)
if err != nil {
return xerrors.Errorf("failed to get proving set for miner: %w", err)
}
p.sset = sset
// Compute how many blocks back we have to look from the given tipset for the PoSt challenge
challengeLookback := int((int64(p.ts.Height()) - int64(p.ppe)) + int64(build.PoStChallangeTime) + int64(build.PoStRandomnessLookback))
r, err := p.m.api.ChainGetRandomness(ctx, p.ts, nil, challengeLookback)
if err != nil {
return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", p.ts.Height(), p.ppe, err)
}
p.r = r
return nil
}
func (p *post) sortedSectorInfo() sectorbuilder.SortedSectorInfo {
sbsi := make([]sectorbuilder.SectorInfo, len(p.sset))
for k, sector := range p.sset {
var commR [sectorbuilder.CommLen]byte
copy(commR[:], sector.CommR)
sbsi[k] = sectorbuilder.SectorInfo{
SectorID: sector.SectorID,
CommR: commR,
}
}
return sectorbuilder.NewSortedSectorInfo(sbsi)
}
func (p *post) runPost(ctx context.Context) error {
ctx, span := trace.StartSpan(ctx, "storage.runPost")
defer span.End()
log.Infow("running PoSt", "delayed-by",
int64(p.ts.Height())-(int64(p.ppe)-int64(build.PoStChallangeTime)),
"chain-random", p.r, "ppe", p.ppe, "height", p.ts.Height())
tsStart := time.Now()
var faults []uint64 // TODO
var seed [32]byte
copy(seed[:], p.r)
proof, err := p.m.sb.GeneratePoSt(p.sortedSectorInfo(), seed, faults)
if err != nil {
return xerrors.Errorf("running post failed: %w", err)
}
elapsed := time.Since(tsStart)
p.proof = proof
log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed)
return nil
}
func (p *post) commitPost(ctx context.Context) error {
ctx, span := trace.StartSpan(ctx, "storage.commitPost")
defer span.End()
params := &actors.SubmitPoStParams{
Proof: p.proof,
DoneSet: types.BitFieldFromSet(nil),
}
enc, aerr := actors.SerializeParams(params)
if aerr != nil {
return xerrors.Errorf("could not serialize submit post parameters: %w", aerr)
}
msg := &types.Message{
To: p.m.maddr,
From: p.m.worker,
Method: actors.MAMethods.SubmitPoSt,
Params: enc,
Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late
GasLimit: types.NewInt(1000000 /* i dont know help */),
GasPrice: types.NewInt(1),
}
log.Info("mpush")
smsg, err := p.m.api.MpoolPushMessage(ctx, msg)
if err != nil {
return xerrors.Errorf("pushing message to mpool: %w", err)
}
p.smsg = smsg.Cid()
return nil
}
func (p *post) waitCommit(ctx context.Context) error {
ctx, span := trace.StartSpan(ctx, "storage.waitPost")
defer span.End()
log.Infof("Waiting for post %s to appear on chain", p.smsg)
// make sure it succeeds...
rec, err := p.m.api.StateWaitMsg(ctx, p.smsg)
if err != nil {
return err
}
if rec.Receipt.ExitCode != 0 {
log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode)
// TODO: Do something
}
return nil
}
func (m *Miner) computePost(ppe uint64) func(ctx context.Context, ts *types.TipSet, curH uint64) error {
called := 0
return func(ts *types.TipSet, curH uint64) error {
return func(ctx context.Context, ts *types.TipSet, curH uint64) error {
called++
if called > 1 {
log.Errorw("BUG: computePost callback called again", "ppe", ppe,
@ -106,77 +264,21 @@ func (m *Miner) computePost(ppe uint64) func(ts *types.TipSet, curH uint64) erro
return nil
}
ctx := context.TODO()
sset, err := m.api.StateMinerProvingSet(ctx, m.maddr, ts)
if err != nil {
return xerrors.Errorf("failed to get proving set for miner: %w", err)
}
r, err := m.api.ChainGetRandomness(ctx, ts, nil, int(int64(ts.Height())-int64(ppe)+int64(build.PoStChallangeTime)+int64(build.PoStRandomnessLookback))) // TODO: review: check math
if err != nil {
return xerrors.Errorf("failed to get chain randomness for post (ts=%d; ppe=%d): %w", ts.Height(), ppe, err)
}
log.Infow("running PoSt", "delayed-by",
int64(ts.Height())-(int64(ppe)-int64(build.PoStChallangeTime)),
"chain-random", r, "ppe", ppe, "height", ts.Height())
tsStart := time.Now()
var faults []uint64
proof, err := m.secst.RunPoSt(ctx, sset, r, faults)
if err != nil {
return xerrors.Errorf("running post failed: %w", err)
}
elapsed := time.Since(tsStart)
log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed)
params := &actors.SubmitPoStParams{
Proof: proof,
DoneSet: types.BitFieldFromSet(nil),
}
enc, aerr := actors.SerializeParams(params)
if aerr != nil {
return xerrors.Errorf("could not serialize submit post parameters: %w", err)
}
msg := &types.Message{
To: m.maddr,
From: m.worker,
Method: actors.MAMethods.SubmitPoSt,
Params: enc,
Value: types.NewInt(1000), // currently hard-coded late fee in actor, returned if not late
GasLimit: types.NewInt(1000000 /* i dont know help */),
GasPrice: types.NewInt(1),
}
log.Info("mpush")
smsg, err := m.api.MpoolPushMessage(ctx, msg)
if err != nil {
return xerrors.Errorf("pushing message to mpool: %w", err)
}
log.Infof("Waiting for post %s to appear on chain", smsg.Cid())
// make sure it succeeds...
rec, err := m.api.StateWaitMsg(ctx, smsg.Cid())
err := (&post{
m: m,
ppe: ppe,
ts: ts,
}).doPost(ctx)
if err != nil {
return err
}
if rec.Receipt.ExitCode != 0 {
log.Warnf("SubmitPoSt EXIT: %d", rec.Receipt.ExitCode)
// TODO: Do something
}
m.scheduleNextPost(ppe + build.ProvingPeriodDuration)
return nil
}
}
func sectorIdList(si []*api.SectorInfo) []uint64 {
func sectorIdList(si []*api.ChainSectorInfo) []uint64 {
out := make([]uint64, len(si))
for i, s := range si {
out[i] = s.SectorID

254
storage/sealing.go Normal file
View File

@ -0,0 +1,254 @@
package storage
import (
"context"
"io"
cid "github.com/ipfs/go-cid"
xerrors "golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error)
type SealTicket struct {
BlockHeight uint64
TicketBytes []byte
}
func (t *SealTicket) SB() sectorbuilder.SealTicket {
out := sectorbuilder.SealTicket{BlockHeight: t.BlockHeight}
copy(out.TicketBytes[:], t.TicketBytes)
return out
}
type SealSeed struct {
BlockHeight uint64
TicketBytes []byte
}
func (t *SealSeed) SB() sectorbuilder.SealSeed {
out := sectorbuilder.SealSeed{BlockHeight: t.BlockHeight}
copy(out.TicketBytes[:], t.TicketBytes)
return out
}
type Piece struct {
DealID uint64
Ref string
Size uint64
CommP []byte
}
func (p *Piece) ppi() (out sectorbuilder.PublicPieceInfo) {
out.Size = p.Size
copy(out.CommP[:], p.CommP)
return out
}
type SectorInfo struct {
State api.SectorState
SectorID uint64
// Packing
Pieces []Piece
// PreCommit
CommC []byte
CommD []byte
CommR []byte
CommRLast []byte
Proof []byte
Ticket SealTicket
PreCommitMessage *cid.Cid
// PreCommitted
Seed SealSeed
// Committing
CommitMessage *cid.Cid
}
type sectorUpdate struct {
newState api.SectorState
id uint64
err error
mut func(*SectorInfo)
}
func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo {
out := make([]sectorbuilder.PublicPieceInfo, len(t.Pieces))
for i, piece := range t.Pieces {
out[i] = piece.ppi()
}
return out
}
func (t *SectorInfo) deals() []uint64 {
out := make([]uint64, len(t.Pieces))
for i, piece := range t.Pieces {
out[i] = piece.DealID
}
return out
}
func (t *SectorInfo) refs() []string {
out := make([]string, len(t.Pieces))
for i, piece := range t.Pieces {
out[i] = piece.Ref
}
return out
}
func (t *SectorInfo) existingPieces() []uint64 {
out := make([]uint64, len(t.Pieces))
for i, piece := range t.Pieces {
out[i] = piece.Size
}
return out
}
func (t *SectorInfo) rspco() sectorbuilder.RawSealPreCommitOutput {
var out sectorbuilder.RawSealPreCommitOutput
copy(out.CommC[:], t.CommC)
copy(out.CommD[:], t.CommD)
copy(out.CommR[:], t.CommR)
copy(out.CommRLast[:], t.CommRLast)
return out
}
func (m *Miner) sectorStateLoop(ctx context.Context) {
// TODO: restore state
go func() {
defer log.Warn("quitting deal provider loop")
defer close(m.stopped)
for {
select {
case sector := <-m.sectorIncoming:
m.onSectorIncoming(sector)
case update := <-m.sectorUpdated:
m.onSectorUpdated(ctx, update)
case <-m.stop:
return
}
}
}()
}
func (m *Miner) onSectorIncoming(sector *SectorInfo) {
has, err := m.sectors.Has(sector.SectorID)
if err != nil {
return
}
if has {
log.Warnf("SealPiece called more than once for sector %d", sector.SectorID)
return
}
if err := m.sectors.Begin(sector.SectorID, sector); err != nil {
// We may have re-sent the proposal
log.Errorf("deal tracking failed: %s", err)
m.failSector(sector.SectorID, err)
return
}
go func() {
select {
case m.sectorUpdated <- sectorUpdate{
newState: api.Packing,
id: sector.SectorID,
}:
case <-m.stop:
log.Warn("failed to send incoming sector update, miner shutting down")
}
}()
}
func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) {
log.Infof("Sector %d updated state to %s", update.id, api.SectorStateStr(update.newState))
var sector SectorInfo
err := m.sectors.Mutate(update.id, func(s *SectorInfo) error {
s.State = update.newState
if update.mut != nil {
update.mut(s)
}
sector = *s
return nil
})
if update.err != nil {
log.Errorf("sector %d failed: %s", update.id, update.err)
m.failSector(update.id, update.err)
return
}
if err != nil {
m.failSector(update.id, err)
return
}
switch update.newState {
case api.Packing:
m.handle(ctx, sector, m.finishPacking, api.Unsealed)
case api.Unsealed:
m.handle(ctx, sector, m.sealPreCommit, api.PreCommitting)
case api.PreCommitting:
m.handle(ctx, sector, m.preCommit, api.PreCommitted)
case api.PreCommitted:
m.handle(ctx, sector, m.preCommitted, api.SectorNoUpdate)
case api.Committing:
m.handle(ctx, sector, m.committing, api.Proving)
case api.SectorNoUpdate: // noop
default:
log.Error("unexpected sector update state: %d", update.newState)
}
}
func (m *Miner) failSector(id uint64, err error) {
log.Errorf("sector %d error: %+v", id, err)
}
func (m *Miner) SealPiece(ctx context.Context, ref string, size uint64, r io.Reader, dealID uint64) (uint64, error) {
log.Infof("Seal piece for deal %d", dealID)
sid, err := m.sb.AcquireSectorId() // TODO: Put more than one thing in a sector
if err != nil {
return 0, xerrors.Errorf("acquiring sector ID: %w", err)
}
ppi, err := m.sb.AddPiece(size, sid, r, []uint64{})
if err != nil {
return 0, xerrors.Errorf("adding piece to sector: %w", err)
}
return sid, m.newSector(ctx, sid, dealID, ref, ppi)
}
func (m *Miner) newSector(ctx context.Context, sid uint64, dealID uint64, ref string, ppi sectorbuilder.PublicPieceInfo) error {
si := &SectorInfo{
SectorID: sid,
Pieces: []Piece{
{
DealID: dealID,
Ref: ref,
Size: ppi.Size,
CommP: ppi.CommP[:],
},
},
}
select {
case m.sectorIncoming <- si:
return nil
case <-ctx.Done():
return xerrors.Errorf("failed to submit sector for sealing, queue full: %w", ctx.Err())
}
}

57
storage/sealing_utils.go Normal file
View File

@ -0,0 +1,57 @@
package storage
import (
"math/bits"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func fillersFromRem(toFill uint64) ([]uint64, error) {
// Convert to in-sector bytes for easier math:
//
// Sector size to user bytes ratio is constant, e.g. for 1024B we have 1016B
// of user-usable data.
//
// (1024/1016 = 128/127)
//
// Given that we can get sector size by simply adding 1/127 of the user
// bytes
//
// (we convert to sector bytes as they are nice round binary numbers)
toFill += toFill / 127
// We need to fill the sector with pieces that are powers of 2. Conveniently
// computers store numbers in binary, which means we can look at 1s to get
// all the piece sizes we need to fill the sector. It also means that number
// of pieces is the number of 1s in the number of remaining bytes to fill
out := make([]uint64, bits.OnesCount64(toFill))
for i := range out {
// Extract the next lowest non-zero bit
next := bits.TrailingZeros64(toFill)
psize := uint64(1) << next
// e.g: if the number is 0b010100, psize will be 0b000100
// set that bit to 0 by XORing it, so the next iteration looks at the
// next bit
toFill ^= psize
// Add the piece size to the list of pieces we need to create
out[i] = sectorbuilder.UserBytesForSectorSize(psize)
}
return out, nil
}
func (m *Miner) ListSectors() ([]SectorInfo, error) {
var sectors []SectorInfo
if err := m.sectors.List(&sectors); err != nil {
return nil, err
}
return sectors, nil
}
func (m *Miner) GetSectorInfo(sid uint64) (SectorInfo, error) {
var out SectorInfo
err := m.sectors.Get(sid, &out)
return out, err
}

46
storage/sealing_utils_test.go Normal file
View File

@ -0,0 +1,46 @@
package storage
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
func testFill(t *testing.T, n uint64, exp []uint64) {
f, err := fillersFromRem(n)
assert.NoError(t, err)
assert.Equal(t, exp, f)
var sum uint64
for _, u := range f {
sum += u
}
assert.Equal(t, n, sum)
}
func TestFillersFromRem(t *testing.T) {
for i := 8; i < 32; i++ {
// single
ub := sectorbuilder.UserBytesForSectorSize(uint64(1) << i)
testFill(t, ub, []uint64{ub})
// 2
ub = sectorbuilder.UserBytesForSectorSize(uint64(5) << i)
ub1 := sectorbuilder.UserBytesForSectorSize(uint64(1) << i)
ub3 := sectorbuilder.UserBytesForSectorSize(uint64(4) << i)
testFill(t, ub, []uint64{ub1, ub3})
// 4
ub = sectorbuilder.UserBytesForSectorSize(uint64(15) << i)
ub2 := sectorbuilder.UserBytesForSectorSize(uint64(2) << i)
ub4 := sectorbuilder.UserBytesForSectorSize(uint64(8) << i)
testFill(t, ub, []uint64{ub1, ub2, ub3, ub4})
// different 2
ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i)
testFill(t, ub, []uint64{ub1, ub4})
}
}

344
storage/sector/store.go
View File

@ -1,344 +0,0 @@
package sector
import (
"context"
"fmt"
"io"
"sync"
"time"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
cbor "github.com/ipfs/go-ipld-cbor"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
func init() {
cbor.RegisterCborType(dealMapping{})
}
var log = logging.Logger("sectorstore")
var sectorDealsPrefix = datastore.NewKey("/sectordeals")
type dealMapping struct {
DealIDs []uint64
Committed bool
}
type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error)
// TODO: eventually handle sector storage here instead of in rust-sectorbuilder
type Store struct {
waitingLk sync.Mutex
sb *sectorbuilder.SectorBuilder
tktFn TicketFn
dealsLk sync.Mutex
deals datastore.Datastore
waiting map[uint64]chan struct{}
incoming []chan sectorbuilder.SectorSealingStatus
// TODO: outdated chan
closeCh chan struct{}
}
func NewStore(sb *sectorbuilder.SectorBuilder, ds dtypes.MetadataDS, tktFn TicketFn) *Store {
return &Store{
sb: sb,
tktFn: tktFn,
deals: namespace.Wrap(ds, sectorDealsPrefix),
waiting: map[uint64]chan struct{}{},
closeCh: make(chan struct{}),
}
}
func (s *Store) Service() {
go s.service()
}
func (s *Store) poll() {
log.Debug("polling for sealed sectors...")
// get a list of sectors to poll
s.waitingLk.Lock()
toPoll := make([]uint64, 0, len(s.waiting))
for id := range s.waiting {
toPoll = append(toPoll, id)
}
s.waitingLk.Unlock()
var done []sectorbuilder.SectorSealingStatus
// check status of each
for _, sec := range toPoll {
status, err := s.sb.SealStatus(sec)
if err != nil {
log.Errorf("getting seal status: %s", err)
continue
}
if status.State == sealing_state.Sealed {
done = append(done, status)
}
}
// send updates
s.waitingLk.Lock()
for _, sector := range done {
watch, ok := s.waiting[sector.SectorID]
if ok {
close(watch)
delete(s.waiting, sector.SectorID)
}
for _, c := range s.incoming {
c <- sector // TODO: ctx!
}
}
s.waitingLk.Unlock()
}
func (s *Store) restartSealing() {
sectors, err := s.sb.GetAllStagedSectors()
if err != nil {
return
}
for _, sid := range sectors {
status, err := s.sb.SealStatus(sid)
if err != nil {
return
}
if status.State != sealing_state.Paused {
continue
}
log.Infof("Sector %d is in paused state, resuming sealing", sid)
go func() {
// TODO: when we refactor wait-for-seal below, care about this output too
// (see SealSector below)
_, err := s.sb.ResumeSealSector(sid)
if err != nil {
return
}
}()
}
}
func (s *Store) service() {
poll := time.Tick(5 * time.Second)
s.restartSealing()
for {
select {
case <-poll:
s.poll()
case <-s.closeCh:
s.waitingLk.Lock()
for _, c := range s.incoming {
close(c)
}
s.waitingLk.Unlock()
return
}
}
}
func (s *Store) AddPiece(ref string, size uint64, r io.Reader, dealIDs ...uint64) (sectorID uint64, err error) {
sectorID, err = s.sb.AddPiece(ref, size, r)
if err != nil {
return 0, err
}
s.waitingLk.Lock()
_, exists := s.waiting[sectorID]
if !exists { // pieces can share sectors
s.waiting[sectorID] = make(chan struct{})
}
s.waitingLk.Unlock()
s.dealsLk.Lock()
defer s.dealsLk.Unlock()
k := datastore.NewKey(fmt.Sprint(sectorID))
e, err := s.deals.Get(k)
var deals dealMapping
switch err {
case nil:
if err := cbor.DecodeInto(e, &deals); err != nil {
return 0, err
}
if deals.Committed {
return 0, xerrors.Errorf("sector %d already committed", sectorID)
}
fallthrough
case datastore.ErrNotFound:
deals.DealIDs = append(deals.DealIDs, dealIDs...)
d, err := cbor.DumpObject(&deals)
if err != nil {
return 0, err
}
if err := s.deals.Put(k, d); err != nil {
return 0, err
}
default:
return 0, err
}
return sectorID, nil
}
func (s *Store) DealsForCommit(sectorID uint64) ([]uint64, error) {
s.dealsLk.Lock()
defer s.dealsLk.Unlock()
k := datastore.NewKey(fmt.Sprint(sectorID))
e, err := s.deals.Get(k)
switch err {
case nil:
var deals dealMapping
if err := cbor.DecodeInto(e, &deals); err != nil {
return nil, err
}
if deals.Committed {
log.Errorf("getting deal IDs for sector %d: sector already marked as committed", sectorID)
}
deals.Committed = true
d, err := cbor.DumpObject(&deals)
if err != nil {
return nil, err
}
if err := s.deals.Put(k, d); err != nil {
return nil, err
}
return deals.DealIDs, nil
case datastore.ErrNotFound:
log.Errorf("getting deal IDs for sector %d failed: %s", err)
return []uint64{}, nil
default:
return nil, err
}
}
func (s *Store) SealSector(ctx context.Context, sectorID uint64) error {
tkt, err := s.tktFn(ctx)
if err != nil {
return err
}
// TODO: That's not async, is it?
// - If not then we probably can drop this wait-for-seal hack below
_, err = s.sb.SealSector(sectorID, *tkt)
if err != nil {
return err
}
return nil
}
func (s *Store) CloseIncoming(c <-chan sectorbuilder.SectorSealingStatus) {
s.waitingLk.Lock()
var at = -1
for i, ch := range s.incoming {
if ch == c {
at = i
}
}
if at == -1 {
s.waitingLk.Unlock()
return
}
if len(s.incoming) > 1 {
last := len(s.incoming) - 1
s.incoming[at] = s.incoming[last]
s.incoming[last] = nil
}
s.incoming = s.incoming[:len(s.incoming)-1]
s.waitingLk.Unlock()
}
func (s *Store) Incoming() <-chan sectorbuilder.SectorSealingStatus {
ch := make(chan sectorbuilder.SectorSealingStatus, 8)
s.waitingLk.Lock()
s.incoming = append(s.incoming, ch)
s.waitingLk.Unlock()
return ch
}
func (s *Store) WaitSeal(ctx context.Context, sector uint64) (sectorbuilder.SectorSealingStatus, error) {
s.waitingLk.Lock()
watch, ok := s.waiting[sector]
s.waitingLk.Unlock()
if ok {
select {
case <-watch:
case <-ctx.Done():
return sectorbuilder.SectorSealingStatus{}, ctx.Err()
}
}
return s.sb.SealStatus(sector)
}
func (s *Store) Sealed() ([]sectorbuilder.SectorSealingStatus, error) {
l, err := s.sb.GetAllStagedSectors()
if err != nil {
return nil, err
}
out := make([]sectorbuilder.SectorSealingStatus, 0)
for _, sid := range l {
status, err := s.sb.SealStatus(sid)
if err != nil {
return nil, err
}
if status.State != sealing_state.Sealed {
continue
}
out = append(out, status)
}
return out, nil
}
func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte, faults []uint64) ([]byte, error) {
sbsi := make([]sectorbuilder.SectorInfo, len(sectors))
for k, sector := range sectors {
var commR [sectorbuilder.CommLen]byte
if copy(commR[:], sector.CommR) != sectorbuilder.CommLen {
return nil, xerrors.Errorf("commR too short, %d bytes", len(sector.CommR))
}
sbsi[k] = sectorbuilder.SectorInfo{
SectorID: sector.SectorID,
CommR: commR,
}
}
ssi := sectorbuilder.NewSortedSectorInfo(sbsi)
var seed [sectorbuilder.CommLen]byte
if copy(seed[:], r) != sectorbuilder.CommLen {
return nil, xerrors.Errorf("random seed too short, %d bytes", len(r))
}
return s.sb.GeneratePoSt(ssi, seed, faults)
}
func (s *Store) Stop() {
close(s.closeCh)
}

231
storage/sector_states.go Normal file
View File

@ -0,0 +1,231 @@
package storage
import (
"context"
"github.com/pkg/errors"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
type providerHandlerFunc func(ctx context.Context, deal SectorInfo) (func(*SectorInfo), error)
func (m *Miner) handle(ctx context.Context, sector SectorInfo, cb providerHandlerFunc, next api.SectorState) {
go func() {
mut, err := cb(ctx, sector)
if err == nil && next == api.SectorNoUpdate {
return
}
select {
case m.sectorUpdated <- sectorUpdate{
newState: next,
id: sector.SectorID,
err: err,
mut: mut,
}:
case <-m.stop:
}
}()
}
func (m *Miner) finishPacking(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) {
log.Infow("performing filling up rest of the sector...", "sector", sector.SectorID)
var allocated uint64
for _, piece := range sector.Pieces {
allocated += piece.Size
}
ubytes := sectorbuilder.UserBytesForSectorSize(m.sb.SectorSize())
if allocated > ubytes {
return nil, xerrors.Errorf("too much data in sector: %d > %d", allocated, ubytes)
}
fillerSizes, err := fillersFromRem(ubytes - allocated)
if err != nil {
return nil, err
}
if len(fillerSizes) > 0 {
log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID)
}
pieces, err := m.storeGarbage(ctx, sector.SectorID, sector.existingPieces(), fillerSizes...)
if err != nil {
return nil, xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err)
}
return func(info *SectorInfo) {
info.Pieces = append(info.Pieces, pieces...)
}, nil
}
func (m *Miner) sealPreCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) {
log.Infow("performing sector replication...", "sector", sector.SectorID)
ticket, err := m.tktFn(ctx)
if err != nil {
return nil, err
}
rspco, err := m.sb.SealPreCommit(sector.SectorID, *ticket, sector.pieceInfos())
if err != nil {
return nil, xerrors.Errorf("seal pre commit failed: %w", err)
}
return func(info *SectorInfo) {
info.CommC = rspco.CommC[:]
info.CommD = rspco.CommD[:]
info.CommR = rspco.CommR[:]
info.CommRLast = rspco.CommRLast[:]
info.Ticket = SealTicket{
BlockHeight: ticket.BlockHeight,
TicketBytes: ticket.TicketBytes[:],
}
}, nil
}
func (m *Miner) preCommit(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) {
params := &actors.SectorPreCommitInfo{
SectorNumber: sector.SectorID,
CommR: sector.CommR,
SealEpoch: sector.Ticket.BlockHeight,
DealIDs: sector.deals(),
}
enc, aerr := actors.SerializeParams(params)
if aerr != nil {
return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr)
}
msg := &types.Message{
To: m.maddr,
From: m.worker,
Method: actors.MAMethods.PreCommitSector,
Params: enc,
Value: types.NewInt(0), // TODO: need to ensure sufficient collateral
GasLimit: types.NewInt(1000000 /* i dont know help */),
GasPrice: types.NewInt(1),
}
log.Info("submitting precommit for sector: ", sector.SectorID)
smsg, err := m.api.MpoolPushMessage(ctx, msg)
if err != nil {
return nil, xerrors.Errorf("pushing message to mpool: %w", err)
}
return func(info *SectorInfo) {
mcid := smsg.Cid()
info.PreCommitMessage = &mcid
}, nil
}
func (m *Miner) preCommitted(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) {
// would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts
log.Info("Sector precommitted: ", sector.SectorID)
mw, err := m.api.StateWaitMsg(ctx, *sector.PreCommitMessage)
if err != nil {
return nil, err
}
if mw.Receipt.ExitCode != 0 {
log.Error("sector precommit failed: ", mw.Receipt.ExitCode)
return nil, err
}
log.Info("precommit message landed on chain: ", sector.SectorID)
randHeight := mw.TipSet.Height() + build.InteractivePoRepDelay - 1 // -1 because of how the messages are applied
log.Infof("precommit for sector %d made it on chain, will start proof computation at height %d", sector.SectorID, randHeight)
err = m.events.ChainAt(func(ctx context.Context, ts *types.TipSet, curH uint64) error {
rand, err := m.api.ChainGetRandomness(ctx, ts, nil, int(ts.Height()-randHeight))
if err != nil {
return xerrors.Errorf("failed to get randomness for computing seal proof: %w", err)
}
m.sectorUpdated <- sectorUpdate{
newState: api.Committing,
id: sector.SectorID,
mut: func(info *SectorInfo) {
info.Seed = SealSeed{
BlockHeight: randHeight,
TicketBytes: rand,
}
},
}
return nil
}, func(ctx context.Context, ts *types.TipSet) error {
log.Warn("revert in interactive commit sector step")
return nil
}, 3, mw.TipSet.Height()+build.InteractivePoRepDelay)
if err != nil {
log.Warn("waitForPreCommitMessage ChainAt errored: ", err)
}
return nil, nil
}
func (m *Miner) committing(ctx context.Context, sector SectorInfo) (func(*SectorInfo), error) {
log.Info("scheduling seal proof computation...")
proof, err := m.sb.SealCommit(sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.refs(), sector.rspco())
if err != nil {
return nil, xerrors.Errorf("computing seal proof failed: %w", err)
}
// TODO: Consider splitting states and persist proof for faster recovery
params := &actors.SectorProveCommitInfo{
Proof: proof,
SectorID: sector.SectorID,
DealIDs: sector.deals(),
}
enc, aerr := actors.SerializeParams(params)
if aerr != nil {
return nil, xerrors.Errorf("could not serialize commit sector parameters: %w", aerr)
}
msg := &types.Message{
To: m.maddr,
From: m.worker,
Method: actors.MAMethods.ProveCommitSector,
Params: enc,
Value: types.NewInt(0), // TODO: need to ensure sufficient collateral
GasLimit: types.NewInt(1000000 /* i dont know help */),
GasPrice: types.NewInt(1),
}
smsg, err := m.api.MpoolPushMessage(ctx, msg)
if err != nil {
log.Error(errors.Wrap(err, "pushing message to mpool"))
}
// TODO: Separate state before this wait, so we persist message cid?
mw, err := m.api.StateWaitMsg(ctx, smsg.Cid())
if err != nil {
return nil, xerrors.Errorf("failed to wait for porep inclusion: %w", err)
}
if mw.Receipt.ExitCode != 0 {
log.Errorf("UNHANDLED: submitting sector proof failed (t:%x; s:%x(%d); p:%x)", sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, params.Proof)
return nil, xerrors.New("UNHANDLED: submitting sector proof failed")
}
m.beginPosting(ctx)
return func(info *SectorInfo) {
mcid := smsg.Cid()
info.CommitMessage = &mcid
info.Proof = proof
}, nil
}

77
storage/sectorblocks/blocks.go
View File

@ -1,25 +1,29 @@
package sectorblocks
import (
"bytes"
"context"
"errors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/ipfs/go-datastore/namespace"
"github.com/ipfs/go-datastore/query"
blockstore "github.com/ipfs/go-ipfs-blockstore"
ipld "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-unixfs"
"io"
"sync"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"github.com/ipfs/go-datastore/query"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
files "github.com/ipfs/go-ipfs-files"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
"github.com/ipfs/go-unixfs"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/storage/sector"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/padreader"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/storage"
)
type SealSerialization uint8
@ -34,7 +38,7 @@ var imBlocksPrefix = datastore.NewKey("/intermediate")
var ErrNotFound = errors.New("not found")
type SectorBlocks struct {
*sector.Store
*storage.Miner
intermediate blockstore.Blockstore // holds intermediate nodes TODO: consider combining with the staging blockstore
@ -43,9 +47,9 @@ type SectorBlocks struct {
keyLk sync.Mutex
}
func NewSectorBlocks(sectst *sector.Store, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks {
func NewSectorBlocks(miner *storage.Miner, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder) *SectorBlocks {
sbc := &SectorBlocks{
Store: sectst,
Miner: miner,
intermediate: blockstore.NewBlockstore(namespace.Wrap(ds, imBlocksPrefix)),
@ -73,14 +77,14 @@ type UnixfsReader interface {
type refStorer struct {
blockReader UnixfsReader
writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint32) error
writeRef func(cid cid.Cid, pieceRef string, offset uint64, size uint64) error
intermediate blockstore.Blockstore
pieceRef string
remaining []byte
}
func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint32) error {
func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, size uint64) error {
st.keyLk.Lock() // TODO: make this multithreaded
defer st.keyLk.Unlock()
@ -89,25 +93,25 @@ func (st *SectorBlocks) writeRef(cid cid.Cid, pieceRef string, offset uint64, si
err = nil
}
if err != nil {
return err
return xerrors.Errorf("getting existing refs: %w", err)
}
var refs []api.SealedRef
var refs api.SealedRefs
if len(v) > 0 {
if err := cbor.DecodeInto(v, &refs); err != nil {
return err
if err := cborutil.ReadCborRPC(bytes.NewReader(v), &refs); err != nil {
return xerrors.Errorf("decoding existing refs: %w", err)
}
}
refs = append(refs, api.SealedRef{
refs.Refs = append(refs.Refs, api.SealedRef{
Piece: pieceRef,
Offset: offset,
Size: size,
})
newRef, err := cbor.DumpObject(&refs)
newRef, err := cborutil.Dump(&refs)
if err != nil {
return err
return xerrors.Errorf("serializing refs: %w", err)
}
return st.keys.Put(dshelp.CidToDsKey(cid), newRef) // TODO: batch somehow
}
@ -128,20 +132,23 @@ func (r *refStorer) Read(p []byte) (n int, err error) {
for {
data, offset, nd, err := r.blockReader.ReadBlock(context.TODO())
if err != nil {
return 0, err
if err == io.EOF {
return 0, io.EOF
}
return 0, xerrors.Errorf("reading block: %w", err)
}
if len(data) == 0 {
// TODO: batch
// TODO: GC
if err := r.intermediate.Put(nd); err != nil {
return 0, err
return 0, xerrors.Errorf("storing intermediate node: %w", err)
}
continue
}
if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint32(len(data))); err != nil {
return 0, err
if err := r.writeRef(nd.Cid(), r.pieceRef, offset, uint64(len(data))); err != nil {
return 0, xerrors.Errorf("writing ref: %w", err)
}
read := copy(p, data)
@ -153,7 +160,7 @@ func (r *refStorer) Read(p []byte) (n int, err error) {
}
}
func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) {
func (st *SectorBlocks) AddUnixfsPiece(ctx context.Context, ref cid.Cid, r UnixfsReader, dealID uint64) (sectorID uint64, err error) {
size, err := r.Size()
if err != nil {
return 0, err
@ -166,7 +173,9 @@ func (st *SectorBlocks) AddUnixfsPiece(ref cid.Cid, r UnixfsReader, dealID uint6
intermediate: st.intermediate,
}
return st.Store.AddPiece(refst.pieceRef, uint64(size), refst, dealID)
pr, psize := padreader.New(r, uint64(size))
return st.Miner.SealPiece(ctx, refst.pieceRef, psize, pr, dealID)
}
func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) {
@ -187,12 +196,12 @@ func (st *SectorBlocks) List() (map[cid.Cid][]api.SealedRef, error) {
return nil, err
}
var refs []api.SealedRef
if err := cbor.DecodeInto(ent.Value, &refs); err != nil {
var refs api.SealedRefs
if err := cborutil.ReadCborRPC(bytes.NewReader(ent.Value), &refs); err != nil {
return nil, err
}
out[refCid] = refs
out[refCid] = refs.Refs
}
return out, nil
@ -207,12 +216,12 @@ func (st *SectorBlocks) GetRefs(k cid.Cid) ([]api.SealedRef, error) { // TODO: t
return nil, err
}
var refs []api.SealedRef
if err := cbor.DecodeInto(ent, &refs); err != nil {
var refs api.SealedRefs
if err := cborutil.ReadCborRPC(bytes.NewReader(ent), &refs); err != nil {
return nil, err
}
return refs, nil
return refs.Refs, nil
}
func (st *SectorBlocks) GetSize(k cid.Cid) (uint64, error) {