cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #666 from filecoin-project/feat/election-post

Browse Source 
Election post restructuring
This commit is contained in:
Whyrusleeping 2019-12-02 15:32:12 -08:00 committed by GitHub
commit dd6e75f49d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
112 changed files with 3591 additions and 1642 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.circleci/config.yml
View File

@ -46,6 +46,17 @@ jobs:
- go/mod-download
- go/mod-tidy-check
build-all:
executor: golang
steps:
- install-deps
- prepare
- go/mod-download
- run: sudo apt-get update
- run: sudo apt-get install npm
- run:
command: make buildall
test:
description: |
Run tests with gotestsum.
@ -166,3 +177,4 @@ workflows:
- test:
codecov-upload: true
- mod-tidy-check
- build-all

4
.gitignore vendored
View File

@ -1,8 +1,12 @@
/lotus
/lotus-storage-miner
/lotus-seed
/pond
/townhall
/fountain
/stats
/bench
/bench.json
/lotuspond/front/node_modules
/lotuspond/front/build
/cmd/lotus-townhall/townhall/node_modules

10
.gitmodules vendored
View File

@ -1,6 +1,4 @@
[submodule "extern/go-bls-sigs"]
path = extern/go-bls-sigs
url = https://github.com/filecoin-project/go-bls-sigs.git
[submodule "extern/go-sectorbuilder"]
path = extern/go-sectorbuilder
url = https://github.com/filecoin-project/go-sectorbuilder
[submodule "extern/filecoin-ffi"]
path = extern/filecoin-ffi
url = https://github.com/filecoin-project/filecoin-ffi.git
branch = master

78
Makefile
View File

@ -11,38 +11,23 @@ endif
MODULES:=
CLEAN:=
BINS:=
## BLS
## FFI
BLS_PATH:=extern/go-bls-sigs/
BLS_DEPS:=libbls_signatures.a libbls_signatures.pc libbls_signatures.h
BLS_DEPS:=$(addprefix $(BLS_PATH),$(BLS_DEPS))
FFI_PATH:=extern/filecoin-ffi/
FFI_DEPS:=libfilecoin.a filecoin.pc filecoin.h
FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS))
$(BLS_DEPS): build/.bls-install ;
$(FFI_DEPS): build/.filecoin-install ;
build/.bls-install: $(BLS_PATH)
$(MAKE) -C $(BLS_PATH) $(BLS_DEPS:$(BLS_PATH)%=%)
build/.filecoin-install: $(FFI_PATH)
$(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%)
@touch $@
MODULES+=$(BLS_PATH)
BUILD_DEPS+=build/.bls-install
CLEAN+=build/.bls-install
## SECTOR BUILDER
SECTOR_BUILDER_PATH:=extern/go-sectorbuilder/
SECTOR_BUILDER_DEPS:=libsector_builder_ffi.a sector_builder_ffi.pc sector_builder_ffi.h
SECTOR_BUILDER_DEPS:=$(addprefix $(SECTOR_BUILDER_PATH),$(SECTOR_BUILDER_DEPS))
$(SECTOR_BUILDER_DEPS): build/.sector-builder-install ;
build/.sector-builder-install: $(SECTOR_BUILDER_PATH)
$(MAKE) -C $(SECTOR_BUILDER_PATH) $(SECTOR_BUILDER_DEPS:$(SECTOR_BUILDER_PATH)%=%)
@touch $@
MODULES+=$(SECTOR_BUILDER_PATH)
BUILD_DEPS+=build/.sector-builder-install
CLEAN+=build/.sector-builder-install
MODULES+=$(FFI_PATH)
BUILD_DEPS+=build/.filecoin-install
CLEAN+=build/.filecoin-install
$(MODULES): build/.update-modules ;
@ -53,7 +38,7 @@ build/.update-modules:
# end git modules
## PROOFS
## MAIN BINARIES
CLEAN+=build/.update-modules
@ -69,7 +54,7 @@ lotus: $(BUILD_DEPS)
go run github.com/GeertJohan/go.rice/rice append --exec lotus -i ./build
.PHONY: lotus
CLEAN+=lotus
BINS+=lotus
lotus-storage-miner: $(BUILD_DEPS)
rm -f lotus-storage-miner
@ -77,17 +62,25 @@ lotus-storage-miner: $(BUILD_DEPS)
go run github.com/GeertJohan/go.rice/rice append --exec lotus-storage-miner -i ./build
.PHONY: lotus-storage-miner
CLEAN+=lotus-storage-miner
BINS+=lotus-storage-miner
build: lotus lotus-storage-miner
.PHONY: build
install:
install -C ./lotus /usr/local/bin/lotus
install -C ./lotus-storage-miner /usr/local/bin/lotus-storage-miner
# TOOLS
lotus-seed: $(BUILD_DEPS)
rm -f lotus-seed
go build -o lotus-seed ./cmd/lotus-seed
go run github.com/GeertJohan/go.rice/rice append --exec lotus-seed -i ./build
.PHONY: lotus-seed
BINS+=lotus-seed
benchmarks:
go run github.com/whyrusleeping/bencher ./... > bench.json
@echo Submitting results
@ -96,8 +89,9 @@ benchmarks:
pond: build
go build -o pond ./lotuspond
(cd lotuspond/front && npm i && npm run build)
(cd lotuspond/front && npm i && CI=false npm run build)
.PHONY: pond
BINS+=pond
townhall:
rm -f townhall
@ -105,28 +99,42 @@ townhall:
(cd ./cmd/lotus-townhall/townhall && npm i && npm run build)
go run github.com/GeertJohan/go.rice/rice append --exec townhall -i ./cmd/lotus-townhall -i ./build
.PHONY: townhall
BINS+=townhall
fountain:
rm -f fountain
go build -o fountain ./cmd/lotus-fountain
go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain
.PHONY: fountain
BINS+=fountain
chainwatch:
rm -f chainwatch
go build -o chainwatch ./cmd/lotus-chainwatch
go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch
.PHONY: chainwatch
BINS+=chainwatch
bench:
rm -f bench
go build -o bench ./cmd/lotus-bench
go run github.com/GeertJohan/go.rice/rice append --exec bench -i ./build
.PHONY: bench
BINS+=bench
stats:
rm -f stats
go build -o stats ./tools/stats
.PHONY: stats
BINS+=stats
# MISC
buildall: $(BINS)
clean:
rm -rf $(CLEAN)
-$(MAKE) -C $(BLS_PATH) clean
-$(MAKE) -C $(SECTOR_BUILDER_PATH) clean
rm -rf $(CLEAN) $(BINS)
-$(MAKE) -C $(FFI_PATH) clean
.PHONY: clean
dist-clean:

13
api/api_full.go
View File

@ -23,7 +23,7 @@ type FullNode interface {
// First message is guaranteed to be of len == 1, and type == 'current'
ChainNotify(context.Context) (<-chan []*store.HeadChange, error)
ChainHead(context.Context) (*types.TipSet, error)
ChainGetRandomness(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error)
ChainGetRandomness(context.Context, types.TipSetKey, int64) ([]byte, error)
ChainGetBlock(context.Context, cid.Cid) (*types.BlockHeader, error)
ChainGetTipSet(context.Context, types.TipSetKey) (*types.TipSet, error)
ChainGetBlockMessages(context.Context, cid.Cid) (*BlockMessages, error)
@ -51,10 +51,7 @@ type FullNode interface {
// miner
MinerRegister(context.Context, address.Address) error
MinerUnregister(context.Context, address.Address) error
MinerAddresses(context.Context) ([]address.Address, error)
MinerCreateBlock(context.Context, address.Address, *types.TipSet, []*types.Ticket, types.ElectionProof, []*types.SignedMessage, uint64) (*types.BlockMsg, error)
MinerCreateBlock(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error)
// // UX ?
@ -102,7 +99,7 @@ type FullNode interface {
StateMinerPower(context.Context, address.Address, *types.TipSet) (MinerPower, error)
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerPeerID(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error)
StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)
StateMinerElectionPeriodStart(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)
StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error)
StatePledgeCollateral(context.Context, *types.TipSet) (types.BigInt, error)
StateWaitMsg(context.Context, cid.Cid) (*MsgWait, error)
@ -229,12 +226,14 @@ type QueryOffer struct {
MinerPeerID peer.ID
}
func (o *QueryOffer) Order() RetrievalOrder {
func (o *QueryOffer) Order(client address.Address) RetrievalOrder {
return RetrievalOrder{
Root: o.Root,
Size: o.Size,
Total: o.MinPrice,
Client: client,
Miner: o.Miner,
MinerPeerID: o.MinerPeerID,
}

6
api/api_storage.go
View File

@ -86,9 +86,9 @@ type SectorInfo struct {
}
type SealedRef struct {
Piece string
Offset uint64
Size uint64
SectorID uint64
Offset uint64
Size uint64
}
type SealedRefs struct {

23
api/cbor_gen.go
View File

@ -137,11 +137,8 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.Piece (string) (string)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Piece)))); err != nil {
return err
}
if _, err := w.Write([]byte(t.Piece)); err != nil {
// t.t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
@ -172,16 +169,16 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Piece (string) (string)
// t.t.SectorID (uint64) (uint64)
{
sval, err := cbg.ReadString(br)
if err != nil {
return err
}
t.Piece = string(sval)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
// t.t.Offset (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)

99
api/struct.go
View File

@ -37,19 +37,19 @@ type FullNodeStruct struct {
CommonStruct
Internal struct {
ChainNotify func(context.Context) (<-chan []*store.HeadChange, error) `perm:"read"`
ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainGetRandomness func(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error) `perm:"read"`
ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"`
ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"`
ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"`
ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"`
ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"`
ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"`
ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"`
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"`
ChainNotify func(context.Context) (<-chan []*store.HeadChange, error) `perm:"read"`
ChainHead func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainGetRandomness func(context.Context, types.TipSetKey, int64) ([]byte, error) `perm:"read"`
ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"`
ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"`
ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"`
ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"`
ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"`
ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"`
ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"`
ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"`
ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"`
ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"`
SyncState func(context.Context) (*SyncState, error) `perm:"read"`
SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"`
@ -61,10 +61,7 @@ type FullNodeStruct struct {
MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"`
MpoolSub func(context.Context) (<-chan MpoolUpdate, error) `perm:"read"`
MinerRegister func(context.Context, address.Address) error `perm:"admin"`
MinerUnregister func(context.Context, address.Address) error `perm:"admin"`
MinerAddresses func(context.Context) ([]address.Address, error) `perm:"write"`
MinerCreateBlock func(context.Context, address.Address, *types.TipSet, []*types.Ticket, types.ElectionProof, []*types.SignedMessage, uint64) (*types.BlockMsg, error) `perm:"write"`
MinerCreateBlock func(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error) `perm:"write"`
WalletNew func(context.Context, string) (address.Address, error) `perm:"write"`
WalletHas func(context.Context, address.Address) (bool, error) `perm:"write"`
@ -87,28 +84,28 @@ type FullNodeStruct struct {
ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"`
ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"`
StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"`
StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"`
StateMinerProvingPeriodEnd func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"`
StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"`
StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"`
StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"`
StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"`
StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"`
StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"`
StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"`
StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"`
StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"`
StateLookupID func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) `perm:"read"`
StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"`
StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"`
StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"`
StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"`
StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"`
StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"`
StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"`
StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"`
StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"`
StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"`
StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"`
StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"`
StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"`
StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"`
StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketParticipants func(context.Context, *types.TipSet) (map[string]actors.StorageParticipantBalance, error) `perm:"read"`
StateMarketDeals func(context.Context, *types.TipSet) (map[string]actors.OnChainDeal, error) `perm:"read"`
StateMarketStorageDeal func(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error) `perm:"read"`
StateLookupID func(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) `perm:"read"`
StateChangedActors func(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) `perm:"read"`
StateGetReceipt func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"`
MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
@ -233,28 +230,16 @@ func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan MpoolUpdate, erro
return c.Internal.MpoolSub(ctx)
}
func (c *FullNodeStruct) MinerRegister(ctx context.Context, addr address.Address) error {
return c.Internal.MinerRegister(ctx, addr)
}
func (c *FullNodeStruct) MinerUnregister(ctx context.Context, addr address.Address) error {
return c.Internal.MinerUnregister(ctx, addr)
}
func (c *FullNodeStruct) MinerAddresses(ctx context.Context) ([]address.Address, error) {
return c.Internal.MinerAddresses(ctx)
}
func (c *FullNodeStruct) MinerCreateBlock(ctx context.Context, addr address.Address, base *types.TipSet, tickets []*types.Ticket, eproof types.ElectionProof, msgs []*types.SignedMessage, ts uint64) (*types.BlockMsg, error) {
return c.Internal.MinerCreateBlock(ctx, addr, base, tickets, eproof, msgs, ts)
func (c *FullNodeStruct) MinerCreateBlock(ctx context.Context, addr address.Address, base *types.TipSet, ticket *types.Ticket, eproof *types.EPostProof, msgs []*types.SignedMessage, height, ts uint64) (*types.BlockMsg, error) {
return c.Internal.MinerCreateBlock(ctx, addr, base, ticket, eproof, msgs, height, ts)
}
func (c *FullNodeStruct) ChainHead(ctx context.Context) (*types.TipSet, error) {
return c.Internal.ChainHead(ctx)
}
func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, ticks []*types.Ticket, lb int) ([]byte, error) {
return c.Internal.ChainGetRandomness(ctx, pts, ticks, lb)
func (c *FullNodeStruct) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, round int64) ([]byte, error) {
return c.Internal.ChainGetRandomness(ctx, pts, round)
}
func (c *FullNodeStruct) ChainGetTipSetByHeight(ctx context.Context, h uint64, ts *types.TipSet) (*types.TipSet, error) {
@ -377,8 +362,8 @@ func (c *FullNodeStruct) StateMinerPeerID(ctx context.Context, m address.Address
return c.Internal.StateMinerPeerID(ctx, m, ts)
}
func (c *FullNodeStruct) StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return c.Internal.StateMinerProvingPeriodEnd(ctx, actor, ts)
func (c *FullNodeStruct) StateMinerElectionPeriodStart(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return c.Internal.StateMinerElectionPeriodStart(ctx, actor, ts)
}
func (c *FullNodeStruct) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {

57
api/test/deals.go
View File

@ -1,26 +1,32 @@
package test
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"testing"
"time"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl"
)
func init() {
logging.SetAllLoggers(logging.LevelInfo)
build.InsecurePoStValidation = true
}
func TestDealFlow(t *testing.T, b APIBuilder) {
os.Setenv("BELLMAN_NO_GPU", "1")
logging.SetAllLoggers(logging.LevelInfo)
ctx := context.Background()
n, sn := b(t, 1, []int{0})
client := n[0].FullNode.(*impl.FullNodeAPI)
@ -36,13 +42,16 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
}
time.Sleep(time.Second)
r := io.LimitReader(rand.New(rand.NewSource(17)), 1000)
data := make([]byte, 1000)
rand.New(rand.NewSource(5)).Read(data)
r := bytes.NewReader(data)
fcid, err := client.ClientImportLocal(ctx, r)
if err != nil {
t.Fatal(err)
}
maddr, err := address.NewFromString("t0101")
maddr, err := miner.ActorAddress(ctx)
if err != nil {
t.Fatal(err)
}
@ -57,7 +66,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
for mine {
time.Sleep(time.Second)
fmt.Println("mining a block now")
if err := n[0].MineOne(ctx); err != nil {
if err := sn[0].MineOne(ctx); err != nil {
t.Fatal(err)
}
}
@ -90,6 +99,42 @@ loop:
time.Sleep(time.Second / 2)
}
// Retrieval
offers, err := client.ClientFindData(ctx, fcid)
if err != nil {
t.Fatal(err)
}
if len(offers) < 1 {
t.Fatal("no offers")
}
rpath, err := ioutil.TempDir("", "lotus-retrieve-test-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rpath)
caddr, err := client.WalletDefaultAddress(ctx)
if err != nil {
t.Fatal(err)
}
err = client.ClientRetrieve(ctx, offers[0].Order(caddr), filepath.Join(rpath, "ret"))
if err != nil {
t.Fatalf("%+v", err)
}
rdata, err := ioutil.ReadFile(filepath.Join(rpath, "ret"))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(rdata, data) {
t.Fatal("wrong data retrieved")
}
mine = false
fmt.Println("shutting down mining")
<-done

4
api/test/mining.go
View File

@ -9,7 +9,7 @@ import (
func (ts *testSuite) testMining(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{0})
apis, sn := ts.makeNodes(t, 1, []int{0})
api := apis[0]
h1, err := api.ChainHead(ctx)
@ -20,7 +20,7 @@ func (ts *testSuite) testMining(t *testing.T) {
require.NoError(t, err)
<-newHeads
err = api.MineOne(ctx)
err = sn[0].MineOne(ctx)
require.NoError(t, err)
<-newHeads

10
api/test/test.go
View File

@ -11,12 +11,12 @@ import (
type TestNode struct {
api.FullNode
MineOne func(context.Context) error
}
type TestStorageNode struct {
api.StorageMiner
MineOne func(context.Context) error
}
// APIBuilder is a function which is invoked in test suite to provide
@ -43,7 +43,7 @@ func TestApis(t *testing.T, b APIBuilder) {
func (ts *testSuite) testVersion(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{})
apis, _ := ts.makeNodes(t, 1, []int{0})
api := apis[0]
v, err := api.Version(ctx)
@ -57,7 +57,7 @@ func (ts *testSuite) testVersion(t *testing.T) {
func (ts *testSuite) testID(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 1, []int{})
apis, _ := ts.makeNodes(t, 1, []int{0})
api := apis[0]
id, err := api.ID(ctx)
@ -69,7 +69,7 @@ func (ts *testSuite) testID(t *testing.T) {
func (ts *testSuite) testConnectTwo(t *testing.T) {
ctx := context.Background()
apis, _ := ts.makeNodes(t, 2, []int{})
apis, _ := ts.makeNodes(t, 2, []int{0})
p, err := apis[0].NetPeers(ctx)
if err != nil {

1
build/bootstrap/bootstrappers.pi
View File

@ -1 +0,0 @@
/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWFCkQdiJEMBVA6RrWq22ZXVFfM41YX8soQ5QVvNFjMJT8

1
build/bootstrap/root.pi
View File

@ -1 +0,0 @@
/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWSw9h3e6YrYZfRWDcir8qMV7ctZG9VmtXwSaP2ntsKXYf

2
build/paramfetch.go
View File

@ -156,6 +156,8 @@ func doFetch(out string, info paramFile) error {
if err != nil {
return err
}
log.Infof("GET %s", url)
req := http.Request{
Method: "GET",
URL: url,

18
build/params_debug.go
View File

@ -5,10 +5,22 @@ package build
import "os"
// Seconds
const BlockDelay = 2
const BlockDelay = 6
// Blocks
const ProvingPeriodDuration uint64 = 40
// FallbackPoStDelay is the number of epochs the miner needs to wait after
// ElectionPeriodStart before starting fallback post computation
//
// Epochs
const FallbackPoStDelay = 10
// SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
// which the miner is slashed
//
// Epochs
const SlashablePowerDelay = 20
// Epochs
const InteractivePoRepDelay = 2
func init() {
os.Setenv("TRUST_PARAMS", "1")

16
build/params_devnet.go
View File

@ -5,5 +5,17 @@ package build
// Seconds
const BlockDelay = 12
// Blocks
const ProvingPeriodDuration uint64 = 300
// FallbackPoStDelay is the number of epochs the miner needs to wait after
// ElectionPeriodStart before starting fallback post computation
//
// Epochs
const FallbackPoStDelay = 1000
// SlashablePowerDelay is the number of epochs after ElectionPeriodStart, after
// which the miner is slashed
//
// Epochs
const SlashablePowerDelay = 2000
// Epochs
const InteractivePoRepDelay = 10

38
build/params_shared.go
View File

@ -13,6 +13,7 @@ const UnixfsChunkSize uint64 = 1 << 20
const UnixfsLinksPerLevel = 1024
var SectorSizes = []uint64{
1 << 10,
16 << 20,
256 << 20,
1 << 30,
@ -30,7 +31,7 @@ func SupportedSectorSize(ssize uint64) bool {
// /////
// Payments
// Blocks
// Epochs
const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours
// /////
@ -39,13 +40,13 @@ const PaymentChannelClosingDelay = 6 * 60 * 2 // six hours
// Seconds
const AllowableClockDrift = BlockDelay * 2
// Blocks
// Epochs
const ForkLengthThreshold = Finality
// Blocks (e)
const BlocksPerEpoch = 5
// Blocks
// Epochs
const Finality = 500
// constants for Weight calculation
@ -56,36 +57,27 @@ const WRatioDen = 2
// /////
// Proofs
// PoStChallangeTime sets the window in which post computation should happen
// Blocks
const PoStChallangeTime = ProvingPeriodDuration - 6
// PoStRandomnessLookback is additional randomness lookback for PoSt computation
// To compute randomness epoch in a given proving period:
// RandH = PPE - PoStChallangeTime - PoStRandomnessLookback
//
// Blocks
const PoStRandomnessLookback = 1
// Blocks
// Epochs
const SealRandomnessLookback = Finality
// Blocks
// Epochs
const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
// 1 / n
const SectorChallengeRatioDiv = 25
const MaxFallbackPostChallengeCount = 10
// /////
// Mining
// Blocks
// Epochs
const EcRandomnessLookback = 300
const PowerCollateralProportion = 5
const PerCapitaCollateralProportion = 1
const CollateralPrecision = 1000
// Blocks
const InteractivePoRepDelay = 10
// /////
// Devnet settings
@ -99,8 +91,8 @@ var InitialReward *big.Int
const FilecoinPrecision = 1_000_000_000_000_000_000
// six years
// Blocks
const HalvingPeriodBlocks = 6 * 365 * 24 * 60 * 2
// Epochs
const HalvingPeriodEpochs = 6 * 365 * 24 * 60 * 2
// TODO: Move other important consts here
@ -118,6 +110,6 @@ func init() {
// Sync
const BadBlockCacheSize = 1 << 15
// assuming 4000 blocks per round, this lets us not lose any messages across a
// assuming 4000 messages per round, this lets us not lose any messages across a
// 10 block reorg.
const BlsSignatureCacheSize = 40000

144
build/proof-params/parameters.json
View File

@ -1,82 +1,82 @@
{
"v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": {
"cid": "QmT22f1Np1GpW29NXD7Zrv3Ae4poMYhmkDjyscqL8QrJXY",
"digest": "989fd8d989e0f7f1fe21bb010cf1b231",
"sector_size": 16777216
},
"v15-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": {
"cid": "QmVqSdc23to4UwduCCb25223rpSccvtcgPMfRKY1qjucDc",
"digest": "c6d258c37243b8544238a98100e3e399",
"sector_size": 16777216
},
"v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": {
"cid": "QmRTCqgokEGTMfWVaSr7qFXTNotmpd2QBEi8RsvSQKmPLz",
"digest": "ff77a5e270afc6e1c7fbc19e48348fac",
"sector_size": 1073741824
},
"v15-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": {
"cid": "QmRssVAXRN3xp9VdSpTq1pNjkob3QiikoFZiM5hqrmh1VU",
"digest": "b41f35ac26224258e366327716a835a4",
"sector_size": 1073741824
},
"v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
"cid": "QmYNVRVzjXkuxJfnHTU5vmEcUBQf8dabXZ4m53SzqMkBv5",
"digest": "d156b685e4a1fe3a1f7230b6a39b5ad4",
"sector_size": 1024
},
"v15-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": {
"cid": "QmaCEcsCFVuepMKdC5WURbr5ucEyLMNGxQaB7HqSnr2KGh",
"digest": "06ff067ac78cdab5d7bbc82170882241",
"sector_size": 1024
},
"v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": {
"cid": "QmVuabRvJ797NwLisGKwRURASGxopBBgg4rfNsbZoSYzAc",
"digest": "0e1ceb79a459a60508f480e5b1fed7ac",
"v19-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.params": {
"cid": "QmZEKhzKbC7SPngjL85ghyuxabPmiEuUh4fpkC7CDK1J5q",
"digest": "1c81338b8afeaae514fd5d6c08c9c6e5",
"sector_size": 268435456
},
"v15-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": {
"cid": "QmdWENZBAbuUty1vVNn9vmvj1XbJ5UC8qzpcVD35s5AJxG",
"digest": "1b755c74b9d6823c014f6a7ef76249f2",
"v19-proof-of-spacetime-election-09ae025de08399327e14f0cb6b4c907b6fe1e8b77046e31de8921bde588de900.vk": {
"cid": "Qmf5XQuM58jNmxudXeZMuZtGGiNzcd56Fiyn9h76wpX5dN",
"digest": "bb0f07b6071cd28e9348223c6a9c46d1",
"sector_size": 268435456
},
"v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.params": {
"cid": "QmZDVpWTw5Eti5pE7N5z1Cmqsw8hPXhUcvG3cQuceK56LH",
"digest": "6aa80306018ea1328f2d6faf8c080734",
"sector_size": 16777216
},
"v15-stacked-proof-of-replication-0c0b444c6f31d11c8e98003cc99a3b938db26b77a296d4253cda8945c234266d.vk": {
"cid": "QmaoXV7iVSJcfZ5qubYy7NBcXDSdnTzxH85d7M4bdDtfGZ",
"digest": "f6832eb736faf2960e920d32e9780b12",
"sector_size": 16777216
},
"v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.params": {
"cid": "QmbUW3a3q5DHBb7Ufk8iSbnSCZgbwpe3serqfwKmcTd11w",
"digest": "64024e461b07c869df2463d33dd28035",
"sector_size": 268435456
},
"v15-stacked-proof-of-replication-967b11bb59be11b7dc6f2b627520ba450a3aa50846dbbf886cb8b735fe25c4e7.vk": {
"cid": "Qme3QgBBE7hUgrK7G9ZfJhzkbvViN5HALFpFduYs5K1piv",
"digest": "32496f4dc434b0ed9ef49cb62497a7d1",
"sector_size": 268435456
},
"v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.params": {
"cid": "QmZzgJmb8WXYDKxS22HDgnoBYcZzXDC7s2c2zsV7kouNZ9",
"digest": "cd91f7ccb2ff57a06f3375946dcbdc68",
"sector_size": 1073741824
},
"v15-stacked-proof-of-replication-d01cd22091627b721c60a3375b5219af653fb9f6928c70aa7400587d396bc07a.vk": {
"cid": "QmRUMVzFnENbvyNb6aN2AJ2dnnewr1ESGA1UQLMVZZdsJM",
"digest": "92fc84b76dbe69c731518aebcb82ac82",
"sector_size": 1073741824
},
"v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.params": {
"cid": "QmSixsGkxJXTAuFfWKy5aEXEDJEnpcb1GkdQVF8TCPWoHy",
"digest": "f8339ae93478ded3840d0bc7efa19953",
"v19-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.params": {
"cid": "QmaHs5CHcSD6QhaGAp4ysJP4HTZaxPxybCGmUVDk3TNocA",
"digest": "727fb2896a5668d04ba6e0ce71eb50d7",
"sector_size": 1024
},
"v15-stacked-proof-of-replication-f464b92d805d03de6e2c20e2530135b2c8ec96045ec58f342d6feb90282bff8a.vk": {
"cid": "QmTMC8hdZ2TkZ9BFuzHzRLM9SuR2PQdUrSAwABeCuHyV2f",
"digest": "f27f08ce1246ee6612c250bb12803ef1",
"v19-proof-of-spacetime-election-4a2342062706429612fac099694f77294e355c6c9265b80feaff12a0268b0a92.vk": {
"cid": "QmVg8mUXMb6MiZQseAyCmHzEgNkPbV72xQoRmdFr1yJA4w",
"digest": "3fdf4e65a7baf1a2bab5b8a717f3379a",
"sector_size": 1024
},
"v19-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.params": {
"cid": "QmYf1532WoeXhy8AoduWNxpBuwn5DEkpU6YFDGCXh1mqBX",
"digest": "983e641f9df01799bc33d5fb3c3020b2",
"sector_size": 1073741824
},
"v19-proof-of-spacetime-election-512f5e6dc00a37fa13c8b0e468188f85957b7bf1ab36d17fb9fe9ed49ae8d657.vk": {
"cid": "QmVig7LUpNSXUcfjkSxUKsFaxqYxEZdaEARWvi14sbihJ4",
"digest": "e1f7a46b60217f1dddf56671dd86e6a7",
"sector_size": 1073741824
},
"v19-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.params": {
"cid": "QmSTF7C6vQbV6qjEQknXpBDuixBkxYeMshX25NonjJxjbi",
"digest": "b434ece6a37e588ca11ed117f1c14325",
"sector_size": 16777216
},
"v19-proof-of-spacetime-election-6c7cbfe7eed40b6c0b23a213a70648770aed65d9ca03ae85451573c18532304b.vk": {
"cid": "QmaFV9n5scuYxKc9QpJnbBUf4sjhkUzpZ7QkoTqL8XsNQz",
"digest": "9c2e40b6635470d3674324b01c9a3976",
"sector_size": 16777216
},
"v19-stacked-proof-of-replication-5a438611c880423c4f5199787cd531b197846ef46af40af69222467ab9073226.params": {
"cid": "QmQb5mc3ksZh2K5GsNwJrkwuHoWXh8adBdJV3qTbFaRvGe",
"digest": "a187287b1d03cd7ec4f1fccd57f3f3d1",
"sector_size": 1024
},
"v19-stacked-proof-of-replication-5a438611c880423c4f5199787cd531b197846ef46af40af69222467ab9073226.vk": {
"cid": "QmYUjFHzeX22dfzv9wQxF4Qn8wS67bqBwr1Wcz2rga9rAZ",
"digest": "747792363f08d7b53be1a2f51f91582a",
"sector_size": 1024
},
"v19-stacked-proof-of-replication-6ae8ae8998ef393ffd171487bc5141fa3642e9fd39e3a7dbada4f6e7bacffb9b.params": {
"cid": "QmRUZqCwYdcVfQ49Z97g2xkJnaSh4b9SHWwfB3kgZiPo9L",
"digest": "b9494e0ae432a0ebde9c8c877c914583",
"sector_size": 16777216
},
"v19-stacked-proof-of-replication-6ae8ae8998ef393ffd171487bc5141fa3642e9fd39e3a7dbada4f6e7bacffb9b.vk": {
"cid": "QmXngrBy74h8LYhYrbBpefXsXQFWLU3WX3LTXnVQnu1Sdc",
"digest": "af2c0d6834fa581b6f507f8266244dfb",
"sector_size": 16777216
},
"v19-stacked-proof-of-replication-d2ca0f634aebcecba88904612ff82f2349b080b1290879f3fba73c1d9a13d84e.params": {
"cid": "QmfNstuJFKnBt4yJHsNfoKahn1LafBdpJju23U5UNZd9Xy",
"digest": "3911d84ca2b86f491bc7c6372d7d9285",
"sector_size": 268435456
},
"v19-stacked-proof-of-replication-d2ca0f634aebcecba88904612ff82f2349b080b1290879f3fba73c1d9a13d84e.vk": {
"cid": "QmQgSRQBbp7udHDp5pNA3GSCjSyXktUHBw15wx9meL4wgc",
"digest": "ecd2a9bdd178b0ebc9110f568fd70d07",
"sector_size": 268435456
},
"v19-stacked-proof-of-replication-f7b95455d6b7a5e967388a97c2ddc01807eff4c1736e84be4554853bf7783105.params": {
"cid": "QmV5mAkhohUHPRWoNtS3Uo4yvmF23CR2u8JxeeCqthMCdX",
"digest": "9306d91c3518b87016d5efc19428b25e",
"sector_size": 1073741824
},
"v19-stacked-proof-of-replication-f7b95455d6b7a5e967388a97c2ddc01807eff4c1736e84be4554853bf7783105.vk": {
"cid": "QmPjwPdUQJmqp2rQuJX2dy57AC9YCwYtMHdpZzW5BG9JWh",
"digest": "a2004fca043da423df51f6c4bb65788c",
"sector_size": 1073741824
}
}

3
build/testing_flags.go Normal file
View File

@ -0,0 +1,3 @@
package build
var InsecurePoStValidation = false

1
chain/actors/actor_init.go
View File

@ -40,7 +40,6 @@ func init() {
type InitActor struct{}
type InitActorState struct {
// TODO: this needs to be a HAMT, its a dumb map for now
AddressMap cid.Cid
NextID uint64

298
chain/actors/actor_miner.go
View File

@ -6,6 +6,8 @@ import (
"encoding/binary"
"fmt"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/address"
@ -29,12 +31,16 @@ type StorageMinerActorState struct {
PreCommittedSectors map[string]*PreCommittedSector
// All sectors this miner has committed.
//
// AMT[sectorID]ffi.PublicSectorInfo
Sectors cid.Cid
// TODO: Spec says 'StagedCommittedSectors', which one is it?
// Sectors this miner is currently mining. It is only updated
// when a PoSt is submitted (not as each new sector commitment is added).
//
// AMT[sectorID]ffi.PublicSectorInfo
ProvingSet cid.Cid
// TODO: these:
@ -54,12 +60,6 @@ type StorageMinerActorState struct {
// These become the currentFaultSet when a PoSt is submitted.
NextFaultSet types.BitField
// Sectors reported during the last PoSt submission as being 'done'.
// The collateral for them is still being held until
// the next PoSt submission in case early sector
// removal penalization is needed.
NextDoneSet types.BitField
// Amount of power this miner has.
Power types.BigInt
@ -69,7 +69,7 @@ type StorageMinerActorState struct {
// The height at which this miner was slashed at.
SlashedAt uint64
ProvingPeriodEnd uint64
ElectionPeriodStart uint64
}
type MinerInfo struct {
@ -117,7 +117,7 @@ type maMethods struct {
Constructor uint64
PreCommitSector uint64
ProveCommitSector uint64
SubmitPoSt uint64
SubmitFallbackPoSt uint64
SlashStorageFault uint64
GetCurrentProvingSet uint64
ArbitrateDeal uint64
@ -133,16 +133,17 @@ type maMethods struct {
CheckMiner uint64
DeclareFaults uint64
SlashConsensusFault uint64
SubmitElectionPoSt uint64
}
var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
var MAMethods = maMethods{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
func (sma StorageMinerActor) Exports() []interface{} {
return []interface{}{
1: sma.StorageMinerConstructor,
2: sma.PreCommitSector,
3: sma.ProveCommitSector,
4: sma.SubmitPoSt,
4: sma.SubmitFallbackPoSt,
//5: sma.SlashStorageFault,
//6: sma.GetCurrentProvingSet,
//7: sma.ArbitrateDeal,
@ -158,6 +159,7 @@ func (sma StorageMinerActor) Exports() []interface{} {
17: sma.CheckMiner,
18: sma.DeclareFaults,
19: sma.SlashConsensusFault,
20: sma.SubmitElectionPoSt,
}
}
@ -333,7 +335,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc)
if err != nil {
return nil, aerrors.Wrap(err, "failed to compute data commitment")
return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs)
}
if ok, err := ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
@ -345,7 +347,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
// Note: There must exist a unique index in the miner's sector set for each
// sector ID. The `faults`, `recovered`, and `done` parameters of the
// SubmitPoSt method express indices into this sector set.
nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, us.Info.CommR, commD)
nssroot, err := AddToSectorSet(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD)
if err != nil {
return nil, err
}
@ -366,7 +368,9 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
if pss.Count == 0 {
self.ProvingSet = self.Sectors
self.ProvingPeriodEnd = vmctx.BlockHeight() + build.ProvingPeriodDuration
// TODO: probably want to wait until the miner is above a certain
// threshold before starting this
self.ElectionPeriodStart = vmctx.BlockHeight()
}
nstate, err := vmctx.Storage().Put(self)
@ -388,23 +392,12 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
return nil, err
}
type SubmitPoStParams struct {
Proof []byte
DoneSet types.BitField
// TODO: once the spec changes finish, we have more work to do here...
type SubmitFallbackPoStParams struct {
Proof []byte
Candidates []types.EPostTicket
}
func ProvingPeriodEnd(setPeriodEnd, height uint64) (uint64, uint64) {
offset := setPeriodEnd % build.ProvingPeriodDuration
period := ((height - offset - 1) / build.ProvingPeriodDuration) + 1
end := (period * build.ProvingPeriodDuration) + offset
return end, period
}
// TODO: this is a dummy method that allows us to plumb in other parts of the
// system for now.
func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext, params *SubmitPoStParams) ([]byte, ActorError) {
func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VMContext, params *SubmitFallbackPoStParams) ([]byte, ActorError) {
oldstate, self, err := loadState(vmctx)
if err != nil {
return nil, err
@ -419,36 +412,28 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
return nil, aerrors.New(1, "not authorized to submit post for miner")
}
currentProvingPeriodEnd, _ := ProvingPeriodEnd(self.ProvingPeriodEnd, vmctx.BlockHeight())
feesRequired := types.NewInt(0)
if currentProvingPeriodEnd > self.ProvingPeriodEnd {
//TODO late fee calc
feesRequired = types.BigAdd(feesRequired, types.NewInt(1000))
}
//TODO temporary sector failure fees
msgVal := vmctx.Message().Value
if msgVal.LessThan(feesRequired) {
return nil, aerrors.New(2, "not enough funds to pay post submission fees")
}
if msgVal.GreaterThan(feesRequired) {
_, err := vmctx.Send(vmctx.Message().From, 0,
types.BigSub(msgVal, feesRequired), nil)
if err != nil {
return nil, aerrors.Wrap(err, "could not refund excess fees")
/*
// TODO: handle fees
msgVal := vmctx.Message().Value
if msgVal.LessThan(feesRequired) {
return nil, aerrors.New(2, "not enough funds to pay post submission fees")
}
}
if msgVal.GreaterThan(feesRequired) {
_, err := vmctx.Send(vmctx.Message().From, 0,
types.BigSub(msgVal, feesRequired), nil)
if err != nil {
return nil, aerrors.Wrap(err, "could not refund excess fees")
}
}
*/
var seed [sectorbuilder.CommLen]byte
{
randHeight := currentProvingPeriodEnd - build.PoStChallangeTime - build.PoStRandomnessLookback
randHeight := self.ElectionPeriodStart + build.FallbackPoStDelay
if vmctx.BlockHeight() <= randHeight {
// TODO: spec, retcode
return nil, aerrors.Newf(1, "submit PoSt called outside submission window (%d < %d)", vmctx.BlockHeight(), randHeight)
return nil, aerrors.Newf(1, "submit fallback PoSt called too early (%d < %d)", vmctx.BlockHeight(), randHeight)
}
rand, err := vmctx.GetRandomness(randHeight)
@ -468,13 +453,13 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
}
var sectorInfos []sectorbuilder.SectorInfo
var sectorInfos []ffi.PublicSectorInfo
if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
var comms [][]byte
if err := cbor.DecodeInto(v.Raw, &comms); err != nil {
return xerrors.New("could not decode comms")
}
si := sectorbuilder.SectorInfo{
si := ffi.PublicSectorInfo{
SectorID: id,
}
commR := comms[0]
@ -491,10 +476,23 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
}
faults := self.CurrentFaultSet.All()
_ = faults
if ok, lerr := sectorbuilder.VerifyPost(vmctx.Context(), mi.SectorSize,
sectorbuilder.NewSortedSectorInfo(sectorInfos), seed, params.Proof,
faults); !ok || lerr != nil {
proverID := vmctx.Message().To // TODO: normalize to ID address
var candidates []sectorbuilder.EPostCandidate
for _, t := range params.Candidates {
var partial [32]byte
copy(partial[:], t.Partial)
candidates = append(candidates, sectorbuilder.EPostCandidate{
PartialTicket: partial,
SectorID: t.SectorID,
SectorChallengeIndex: t.ChallengeIndex,
})
}
if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID); !ok || lerr != nil {
if lerr != nil {
// TODO: study PoST errors
return nil, aerrors.Absorb(lerr, 4, "PoST error")
@ -503,60 +501,12 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
return nil, aerrors.New(4, "PoST invalid")
}
}
self.CurrentFaultSet = self.NextFaultSet
self.NextFaultSet = types.NewBitField()
ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
if lerr != nil {
return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
}
if err := ss.BatchDelete(params.DoneSet.All()); err != nil {
// TODO: this could fail for system reasons (block not found) or for
// bad user input reasons (e.g. bad doneset). The latter should be a
// non-fatal error
return nil, aerrors.HandleExternalError(err, "failed to delete sectors in done set")
}
self.ProvingSet, lerr = ss.Flush()
if lerr != nil {
return nil, aerrors.HandleExternalError(lerr, "could not flush AMT")
}
oldPower := self.Power
self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))),
types.NewInt(mi.SectorSize))
delta := types.BigSub(self.Power, oldPower)
if self.SlashedAt != 0 {
self.SlashedAt = 0
delta = self.Power
}
prevPE := self.ProvingPeriodEnd
if !self.Active {
self.Active = true
prevPE = 0
}
enc, err := SerializeParams(&UpdateStorageParams{
Delta: delta,
NextProvingPeriodEnd: currentProvingPeriodEnd + build.ProvingPeriodDuration,
PreviousProvingPeriodEnd: prevPE,
})
if err != nil {
// Post submission is successful!
if err := onSuccessfulPoSt(self, vmctx); err != nil {
return nil, err
}
_, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc)
if err != nil {
return nil, err
}
self.ProvingSet = self.Sectors
self.ProvingPeriodEnd = currentProvingPeriodEnd + build.ProvingPeriodDuration
self.NextDoneSet = params.DoneSet
c, err := vmctx.Storage().Put(self)
if err != nil {
return nil, err
@ -586,8 +536,8 @@ func SectorIsUnique(ctx context.Context, s types.Storage, sroot cid.Cid, sid uin
return !found, nil
}
func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) {
ssr, err := amt.LoadAMT(types.WrapStorage(s), ss)
func AddToSectorSet(ctx context.Context, blks amt.Blocks, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) {
ssr, err := amt.LoadAMT(blks, ss)
if err != nil {
return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node")
}
@ -749,7 +699,7 @@ func (sma StorageMinerActor) GetSectorSize(act *types.Actor, vmctx types.VMConte
}
func isLate(height uint64, self *StorageMinerActorState) bool {
return self.ProvingPeriodEnd > 0 && height >= self.ProvingPeriodEnd // TODO: review: maybe > ?
return self.ElectionPeriodStart > 0 && height >= self.ElectionPeriodStart+build.SlashablePowerDelay
}
func (sma StorageMinerActor) IsSlashed(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
@ -817,32 +767,34 @@ type DeclareFaultsParams struct {
}
func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
oldstate, self, aerr := loadState(vmctx)
if aerr != nil {
return nil, aerr
}
challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime
if vmctx.BlockHeight() < challengeHeight {
// TODO: optimized bitfield methods
for _, v := range params.Faults.All() {
self.CurrentFaultSet.Set(v)
/*
oldstate, self, aerr := loadState(vmctx)
if aerr != nil {
return nil, aerr
}
} else {
for _, v := range params.Faults.All() {
self.NextFaultSet.Set(v)
challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime
if vmctx.BlockHeight() < challengeHeight {
// TODO: optimized bitfield methods
for _, v := range params.Faults.All() {
self.CurrentFaultSet.Set(v)
}
} else {
for _, v := range params.Faults.All() {
self.NextFaultSet.Set(v)
}
}
}
nstate, err := vmctx.Storage().Put(self)
if err != nil {
return nil, err
}
if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
return nil, err
}
nstate, err := vmctx.Storage().Put(self)
if err != nil {
return nil, err
}
if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
return nil, err
}
*/
return nil, nil
}
@ -892,6 +844,88 @@ func (sma StorageMinerActor) SlashConsensusFault(act *types.Actor, vmctx types.V
return nil, nil
}
func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, aerrors.ActorError) {
if vmctx.Message().From != NetworkAddress {
return nil, aerrors.Newf(1, "submit election post can only be called by the storage power actor")
}
oldstate, self, aerr := loadState(vmctx)
if aerr != nil {
return nil, aerr
}
if self.SlashedAt != 0 {
return nil, aerrors.New(1, "slashed miners can't perform election PoSt")
}
if err := onSuccessfulPoSt(self, vmctx); err != nil {
return nil, err
}
ncid, err := vmctx.Storage().Put(self)
if err != nil {
return nil, err
}
if err := vmctx.Storage().Commit(oldstate, ncid); err != nil {
return nil, err
}
return nil, nil
}
func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
// TODO: some sector upkeep stuff that is very haphazard and unclear in the spec
var mi MinerInfo
if err := vmctx.Storage().Get(self.Info, &mi); err != nil {
return err
}
pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
if nerr != nil {
return aerrors.HandleExternalError(nerr, "failed to load proving set")
}
self.CurrentFaultSet = self.NextFaultSet
self.NextFaultSet = types.NewBitField()
faults := []uint64{} // TODO
oldPower := self.Power
self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))),
types.NewInt(mi.SectorSize))
delta := types.BigSub(self.Power, oldPower)
if self.SlashedAt != 0 {
self.SlashedAt = 0
delta = self.Power
}
prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay
if !self.Active {
self.Active = true
prevSlashingDeadline = 0
}
enc, err := SerializeParams(&UpdateStorageParams{
Delta: delta,
NextProvingPeriodEnd: vmctx.BlockHeight() + build.SlashablePowerDelay,
PreviousProvingPeriodEnd: prevSlashingDeadline,
})
if err != nil {
return err
}
_, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc)
if err != nil {
return err
}
self.ProvingSet = self.Sectors
self.ElectionPeriodStart = vmctx.BlockHeight()
return nil
}
func slasherShare(total types.BigInt, elapsed uint64) types.BigInt {
// [int(pow(1.26, n) * 10) for n in range(30)]
fracs := []uint64{10, 12, 15, 20, 25, 31, 40, 50, 63, 80, 100, 127, 160, 201, 254, 320, 403, 508, 640, 807, 1017, 1281, 1614, 2034, 2563, 3230, 4070, 5128, 6462, 8142}

26
chain/actors/actor_storagemarket.go
View File

@ -3,16 +3,17 @@ package actors
import (
"bytes"
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-amt-ipld"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-hamt-ipld"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
@ -110,6 +111,15 @@ func (sdp *StorageDealProposal) Sign(ctx context.Context, sign SignFunc) error {
return nil
}
func (sdp *StorageDealProposal) Cid() (cid.Cid, error) {
nd, err := cborutil.AsIpld(sdp)
if err != nil {
return cid.Undef, err
}
return nd.Cid(), nil
}
func (sdp *StorageDealProposal) Verify() error {
unsigned := *sdp
unsigned.ProposerSignature = nil
@ -538,9 +548,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx
return nil, nil
}
// todo: check math (written on a plane, also tired)
// TODO: division is hard, this more than likely has some off-by-one issue
toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.ProvingPeriodDuration))
toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay))
b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker)
if aerr != nil {
@ -603,18 +611,22 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type
return nil, aerrors.HandleExternalError(err, "loading deals amt")
}
if len(params.DealIDs) == 0 {
return nil, aerrors.New(3, "no deal IDs")
}
var pieces []sectorbuilder.PublicPieceInfo
for _, deal := range params.DealIDs {
var dealInfo OnChainDeal
if err := deals.Get(deal, &dealInfo); err != nil {
if _, is := err.(*amt.ErrNotFound); is {
return nil, aerrors.New(3, "deal not found")
return nil, aerrors.New(4, "deal not found")
}
return nil, aerrors.HandleExternalError(err, "getting deal info failed")
}
if dealInfo.Deal.Proposal.Provider != vmctx.Message().From {
return nil, aerrors.New(4, "referenced deal was not from caller")
return nil, aerrors.New(5, "referenced deal was not from caller")
}
var commP [32]byte
@ -628,7 +640,7 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type
commd, err := sectorbuilder.GenerateDataCommitment(params.SectorSize, pieces)
if err != nil {
return nil, aerrors.Absorb(err, 5, "failed to generate data commitment from pieces")
return nil, aerrors.Absorb(err, 6, "failed to generate data commitment from pieces")
}
return commd[:], nil

6
chain/actors/actor_storagepower.go
View File

@ -289,8 +289,8 @@ func (spa StoragePowerActor) UpdateStorage(act *types.Actor, vmctx types.VMConte
self.TotalStorage = types.BigAdd(self.TotalStorage, params.Delta)
previousBucket := params.PreviousProvingPeriodEnd % build.ProvingPeriodDuration
nextBucket := params.NextProvingPeriodEnd % build.ProvingPeriodDuration
previousBucket := params.PreviousProvingPeriodEnd % build.SlashablePowerDelay
nextBucket := params.NextProvingPeriodEnd % build.SlashablePowerDelay
if previousBucket == nextBucket && params.PreviousProvingPeriodEnd != 0 {
nroot, err := vmctx.Storage().Put(&self)
@ -601,7 +601,7 @@ func (spa StoragePowerActor) CheckProofSubmissions(act *types.Actor, vmctx types
}
func checkProofSubmissionsAtH(vmctx types.VMContext, self *StoragePowerState, height uint64) aerrors.ActorError {
bucketID := height % build.ProvingPeriodDuration
bucketID := height % build.SlashablePowerDelay
buckets, eerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingBuckets)
if eerr != nil {

2
chain/actors/actor_storagepower_test.go
View File

@ -170,5 +170,5 @@ func signBlock(t *testing.T, w *wallet.Wallet, worker address.Address, blk *type
t.Fatal(err)
}
blk.BlockSig = *sig
blk.BlockSig = sig
}

62
chain/actors/cbor_gen.go
View File

@ -198,7 +198,7 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{139}); err != nil {
if _, err := w.Write([]byte{138}); err != nil {
return err
}
@ -258,11 +258,6 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.NextDoneSet (types.BitField) (struct)
if err := t.NextDoneSet.MarshalCBOR(w); err != nil {
return err
}
// t.t.Power (types.BigInt) (struct)
if err := t.Power.MarshalCBOR(w); err != nil {
return err
@ -278,8 +273,8 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.ProvingPeriodEnd (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProvingPeriodEnd))); err != nil {
// t.t.ElectionPeriodStart (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ElectionPeriodStart))); err != nil {
return err
}
return nil
@ -296,7 +291,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 11 {
if extra != 10 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -406,15 +401,6 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
return err
}
}
// t.t.NextDoneSet (types.BitField) (struct)
{
if err := t.NextDoneSet.UnmarshalCBOR(br); err != nil {
return err
}
}
// t.t.Power (types.BigInt) (struct)
@ -452,7 +438,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("wrong type for uint64 field")
}
t.SlashedAt = uint64(extra)
// t.t.ProvingPeriodEnd (uint64) (uint64)
// t.t.ElectionPeriodStart (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
@ -461,7 +447,7 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ProvingPeriodEnd = uint64(extra)
t.ElectionPeriodStart = uint64(extra)
return nil
}
@ -827,7 +813,7 @@ func (t *MinerInfo) UnmarshalCBOR(r io.Reader) error {
return nil
}
func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error {
func (t *SubmitFallbackPoStParams) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
@ -844,14 +830,19 @@ func (t *SubmitPoStParams) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.DoneSet (types.BitField) (struct)
if err := t.DoneSet.MarshalCBOR(w); err != nil {
// t.t.Candidates ([]types.EPostTicket) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Candidates)))); err != nil {
return err
}
for _, v := range t.Candidates {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
return nil
}
func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error {
func (t *SubmitFallbackPoStParams) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
@ -883,15 +874,32 @@ func (t *SubmitPoStParams) UnmarshalCBOR(r io.Reader) error {
if _, err := io.ReadFull(br, t.Proof); err != nil {
return err
}
// t.t.DoneSet (types.BitField) (struct)
// t.t.Candidates ([]types.EPostTicket) (slice)
{
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Candidates: array too large (%d)", extra)
}
if err := t.DoneSet.UnmarshalCBOR(br); err != nil {
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Candidates = make([]types.EPostTicket, extra)
}
for i := 0; i < int(extra); i++ {
var v types.EPostTicket
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Candidates[i] = v
}
return nil
}

11
chain/address/address.go
View File

@ -7,7 +7,7 @@ import (
"io"
"strconv"
"github.com/filecoin-project/go-bls-sigs"
bls "github.com/filecoin-project/filecoin-ffi"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/minio/blake2b-simd"
"github.com/multiformats/go-varint"
@ -404,3 +404,12 @@ func (a *Address) UnmarshalCBOR(br io.Reader) error {
return nil
}
func IDFromAddress(addr Address) (uint64, error) {
if addr.Protocol() != ID {
return 0, xerrors.Errorf("cannot get id from non id address")
}
i, _, err := varint.FromUvarint(addr.Payload())
return i, err
}

8
chain/address/address_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"time"
"github.com/filecoin-project/go-bls-sigs"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/multiformats/go-varint"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -287,7 +287,7 @@ func TestVectorActorAddress(t *testing.T) {
func TestRandomBLSAddress(t *testing.T) {
assert := assert.New(t)
pk := bls.PrivateKeyPublicKey(bls.PrivateKeyGenerate())
pk := ffi.PrivateKeyPublicKey(ffi.PrivateKeyGenerate())
addr, err := NewBLSAddress(pk[:])
assert.NoError(err)
@ -412,8 +412,8 @@ func TestInvalidByteAddresses(t *testing.T) {
{append([]byte{2}, make([]byte, PayloadHashLength+1)...), ErrInvalidPayload},
// BLS Protocol
{append([]byte{3}, make([]byte, bls.PublicKeyBytes-1)...), ErrInvalidPayload},
{append([]byte{3}, make([]byte, bls.PrivateKeyBytes+1)...), ErrInvalidPayload},
{append([]byte{3}, make([]byte, ffi.PublicKeyBytes-1)...), ErrInvalidPayload},
{append([]byte{3}, make([]byte, ffi.PrivateKeyBytes+1)...), ErrInvalidPayload},
}
for _, tc := range testCases {

18
chain/deals/cbor_gen.go
View File

@ -857,7 +857,7 @@ func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{129}); err != nil {
if _, err := w.Write([]byte{130}); err != nil {
return err
}
@ -867,6 +867,10 @@ func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
}
// t.t.DealID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
return err
}
return nil
}
@ -881,7 +885,7 @@ func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 1 {
if extra != 2 {
return fmt.Errorf("cbor input had wrong number of fields")
}
@ -897,5 +901,15 @@ func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
t.Proposal = c
}
// t.t.DealID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.DealID = uint64(extra)
return nil
}

11
chain/deals/provider.go
View File

@ -24,6 +24,8 @@ import (
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
var ProviderDsPrefix = "/deals/provider"
type MinerDeal struct {
Client peer.ID
Proposal actors.StorageDealProposal
@ -110,7 +112,7 @@ func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks
actor: minerAddress,
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client"))),
deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))),
ds: ds,
}
@ -222,10 +224,13 @@ func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState da
// data transfer events for opening and progress do not affect deal state
var next api.DealState
var err error
var mut func(*MinerDeal)
switch event {
case datatransfer.Complete:
next = api.DealStaged
err = nil
mut = func(deal *MinerDeal) {
deal.DealID = voucher.DealID
}
case datatransfer.Error:
next = api.DealFailed
err = ErrDataTransferFailed
@ -239,7 +244,7 @@ func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState da
newState: next,
id: voucher.Proposal,
err: err,
mut: nil,
mut: mut,
}:
case <-p.stop:
}

14
chain/deals/provider_states.go
View File

@ -132,10 +132,10 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
return nil, err
}
if len(resp.DealIDs) != 1 {
return nil, xerrors.Errorf("got unexpected number of DealIDs from")
return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA")
}
log.Info("fetching data for a deal")
log.Infof("fetching data for a deal %d", resp.DealIDs[0])
mcid := smsg.Cid()
err = p.sendSignedResponse(&Response{
State: api.DealAccepted,
@ -164,14 +164,12 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
// (see onDataTransferEvent)
_, err = p.dataTransfer.OpenPullDataChannel(ctx,
deal.Client,
&StorageDataTransferVoucher{Proposal: deal.ProposalCid},
&StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]},
deal.Ref,
allSelector,
)
return func(deal *MinerDeal) {
deal.DealID = resp.DealIDs[0]
}, nil
return nil, nil
}
// STAGED
@ -204,11 +202,11 @@ func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
}
sectorID, err := p.secb.AddUnixfsPiece(ctx, deal.Ref, uf, deal.DealID)
sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
if err != nil {
return nil, xerrors.Errorf("AddPiece failed: %s", err)
}
log.Warnf("New Sector: %d", sectorID)
log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
return func(deal *MinerDeal) {
deal.SectorID = sectorID

20
chain/deals/request_validation_test.go
View File

@ -127,7 +127,7 @@ func TestClientRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, pieceRef, nil), deals.ErrNoDeal) {
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Pull should fail if there is no deal stored")
}
})
@ -144,7 +144,7 @@ func TestClientRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, pieceRef, nil), deals.ErrWrongPeer) {
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
t.Fatal("Pull should fail if miner address is incorrect")
}
})
@ -156,7 +156,7 @@ func TestClientRequestValidation(t *testing.T) {
if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Pull should fail if piece ref is incorrect")
}
})
@ -172,7 +172,7 @@ func TestClientRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, pieceRef, nil), deals.ErrInacceptableDealState) {
if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
t.Fatal("Pull should fail if deal is in a state that cannot be data transferred")
}
})
@ -188,7 +188,7 @@ func TestClientRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, pieceRef, nil) != nil {
if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil {
t.Fatal("Pull should should succeed when all parameters are correct")
}
})
@ -220,7 +220,7 @@ func TestProviderRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid()}, pieceRef, nil), deals.ErrNoDeal) {
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
t.Fatal("Push should fail if there is no deal stored")
}
})
@ -237,7 +237,7 @@ func TestProviderRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, pieceRef, nil), deals.ErrWrongPeer) {
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
t.Fatal("Push should fail if miner address is incorrect")
}
})
@ -249,7 +249,7 @@ func TestProviderRequestValidation(t *testing.T) {
if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
t.Fatal("deal tracking failed")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
t.Fatal("Push should fail if piece ref is incorrect")
}
})
@ -265,7 +265,7 @@ func TestProviderRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, pieceRef, nil), deals.ErrInacceptableDealState) {
if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
t.Fatal("Push should fail if deal is in a state that cannot be data transferred")
}
})
@ -281,7 +281,7 @@ func TestProviderRequestValidation(t *testing.T) {
if err != nil {
t.Fatal("unable to construct piece cid")
}
if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, pieceRef, nil) != nil {
if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil {
t.Fatal("Push should should succeed when all parameters are correct")
}
})

1
chain/deals/types.go
View File

@ -90,6 +90,7 @@ type AskResponse struct {
// used by the storage market
type StorageDataTransferVoucher struct {
Proposal cid.Cid
DealID uint64
}
// ToBytes converts the StorageDataTransferVoucher to raw bytes

10
chain/events/events_test.go
View File

@ -55,31 +55,31 @@ func (fcs *fakeCS) ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet
func makeTs(t *testing.T, h uint64, msgcid cid.Cid) *types.TipSet {
a, _ := address.NewFromString("t00")
b, _ := address.NewFromString("t02")
ts, err := types.NewTipSet([]*types.BlockHeader{
var ts, err = types.NewTipSet([]*types.BlockHeader{
{
Height: h,
Miner: a,
Tickets: []*types.Ticket{{[]byte{byte(h % 2)}}},
Ticket: &types.Ticket{VRFProof: []byte{byte(h % 2)}},
ParentStateRoot: dummyCid,
Messages: msgcid,
ParentMessageReceipts: dummyCid,
BlockSig: types.Signature{Type: types.KTBLS},
BlockSig: &types.Signature{Type: types.KTBLS},
BLSAggregate: types.Signature{Type: types.KTBLS},
},
{
Height: h,
Miner: b,
Tickets: []*types.Ticket{{[]byte{byte((h + 1) % 2)}}},
Ticket: &types.Ticket{VRFProof: []byte{byte((h + 1) % 2)}},
ParentStateRoot: dummyCid,
Messages: msgcid,
ParentMessageReceipts: dummyCid,
BlockSig: types.Signature{Type: types.KTBLS},
BlockSig: &types.Signature{Type: types.KTBLS},
BLSAggregate: types.Signature{Type: types.KTBLS},
},
})

4
chain/events/tscache_test.go
View File

@ -27,7 +27,7 @@ func TestTsCache(t *testing.T) {
ParentStateRoot: dummyCid,
Messages: dummyCid,
ParentMessageReceipts: dummyCid,
BlockSig: types.Signature{Type: types.KTBLS},
BlockSig: &types.Signature{Type: types.KTBLS},
BLSAggregate: types.Signature{Type: types.KTBLS},
}})
if err != nil {
@ -69,7 +69,7 @@ func TestTsCacheNulls(t *testing.T) {
ParentStateRoot: dummyCid,
Messages: dummyCid,
ParentMessageReceipts: dummyCid,
BlockSig: types.Signature{Type: types.KTBLS},
BlockSig: &types.Signature{Type: types.KTBLS},
BLSAggregate: types.Signature{Type: types.KTBLS},
}})
if err != nil {

276
chain/gen/gen.go
View File

@ -3,14 +3,20 @@ package gen
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"fmt"
"io/ioutil"
"sync/atomic"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-car"
offline "github.com/ipfs/go-ipfs-exchange-offline"
"github.com/ipfs/go-merkledag"
peer "github.com/libp2p/go-libp2p-peer"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
@ -20,6 +26,9 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/node/repo"
block "github.com/ipfs/go-block-format"
@ -46,12 +55,12 @@ type ChainGen struct {
genesis *types.BlockHeader
CurTipset *store.FullTipSet
Timestamper func(*types.TipSet, int) uint64
Timestamper func(*types.TipSet, uint64) uint64
w *wallet.Wallet
eppProvs map[address.Address]ElectionPoStProver
Miners []address.Address
mworkers []address.Address
receivers []address.Address
banker address.Address
bankerNonce uint64
@ -102,16 +111,6 @@ func NewGenerator() (*ChainGen, error) {
return nil, xerrors.Errorf("creating memrepo wallet failed: %w", err)
}
worker1, err := w.GenerateKey(types.KTBLS)
if err != nil {
return nil, xerrors.Errorf("failed to generate worker key: %w", err)
}
worker2, err := w.GenerateKey(types.KTBLS)
if err != nil {
return nil, xerrors.Errorf("failed to generate worker key: %w", err)
}
banker, err := w.GenerateKey(types.KTSecp256k1)
if err != nil {
return nil, xerrors.Errorf("failed to generate banker key: %w", err)
@ -125,16 +124,58 @@ func NewGenerator() (*ChainGen, error) {
}
}
maddr1, err := address.NewFromString("t0300")
if err != nil {
return nil, err
}
m1temp, err := ioutil.TempDir("", "preseal")
if err != nil {
return nil, err
}
genm1, err := seed.PreSeal(maddr1, 1024, 1, m1temp, []byte("some randomness"))
if err != nil {
return nil, err
}
maddr2, err := address.NewFromString("t0301")
if err != nil {
return nil, err
}
m2temp, err := ioutil.TempDir("", "preseal")
if err != nil {
return nil, err
}
genm2, err := seed.PreSeal(maddr2, 1024, 1, m2temp, []byte("some randomness"))
if err != nil {
return nil, err
}
mk1, err := w.Import(&genm1.Key)
if err != nil {
return nil, err
}
mk2, err := w.Import(&genm2.Key)
if err != nil {
return nil, err
}
minercfg := &GenMinerCfg{
Workers: []address.Address{worker1, worker2},
Owners: []address.Address{worker1, worker2},
PeerIDs: []peer.ID{"peerID1", "peerID2"},
PreSeals: map[string]genesis.GenesisMiner{
maddr1.String(): *genm1,
maddr2.String(): *genm2,
},
MinerAddrs: []address.Address{maddr1, maddr2},
}
genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{
worker1: types.FromFil(40000),
worker2: types.FromFil(40000),
banker: types.FromFil(50000),
mk1: types.FromFil(40000),
mk2: types.FromFil(40000),
banker: types.FromFil(50000),
}, minercfg, 100000)
if err != nil {
return nil, xerrors.Errorf("make genesis block failed: %w", err)
@ -153,6 +194,11 @@ func NewGenerator() (*ChainGen, error) {
return nil, xerrors.Errorf("MakeGenesisBlock failed to set miner address")
}
mgen := make(map[address.Address]ElectionPoStProver)
for _, m := range minercfg.MinerAddrs {
mgen[m] = &eppProvider{}
}
sm := stmgr.NewStateManager(cs)
gen := &ChainGen{
@ -164,7 +210,7 @@ func NewGenerator() (*ChainGen, error) {
w: w,
Miners: minercfg.MinerAddrs,
mworkers: minercfg.Workers,
eppProvs: mgen,
banker: banker,
receivers: receievers,
@ -189,20 +235,15 @@ func (cg *ChainGen) GenesisCar() ([]byte, error) {
out := new(bytes.Buffer)
if err := car.WriteCar(context.TODO(), dserv, []cid.Cid{cg.Genesis().Cid()}, out); err != nil {
return nil, err
return nil, xerrors.Errorf("genesis car write car failed: %w", err)
}
return out.Bytes(), nil
}
func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m address.Address, ticks []*types.Ticket) (types.ElectionProof, *types.Ticket, error) {
func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m address.Address, round int64) (*types.EPostProof, *types.Ticket, error) {
var lastTicket *types.Ticket
if len(ticks) == 0 {
lastTicket = pts.MinTicket()
} else {
lastTicket = ticks[len(ticks)-1]
}
lastTicket := pts.MinTicket()
st := pts.ParentState()
@ -211,7 +252,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add
return nil, nil, xerrors.Errorf("get miner worker: %w", err)
}
vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, lastTicket.VRFProof)
vrfout, err := ComputeVRF(ctx, cg.w.Sign, worker, m, DSepTicket, lastTicket.VRFProof)
if err != nil {
return nil, nil, xerrors.Errorf("compute VRF: %w", err)
}
@ -220,7 +261,7 @@ func (cg *ChainGen) nextBlockProof(ctx context.Context, pts *types.TipSet, m add
VRFProof: vrfout,
}
win, eproof, err := IsRoundWinner(ctx, pts, append(ticks, tick), m, &mca{w: cg.w, sm: cg.sm})
win, eproof, err := IsRoundWinner(ctx, pts, round, m, cg.eppProvs[m], &mca{w: cg.w, sm: cg.sm})
if err != nil {
return nil, nil, xerrors.Errorf("checking round winner failed: %w", err)
}
@ -248,23 +289,21 @@ func (cg *ChainGen) NextTipSet() (*MinedTipSet, error) {
func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Address) (*MinedTipSet, error) {
var blks []*types.FullBlock
ticketSets := make([][]*types.Ticket, len(miners))
msgs, err := cg.getRandomMessages()
if err != nil {
return nil, xerrors.Errorf("get random messages: %w", err)
}
for len(blks) == 0 {
for i, m := range miners {
proof, t, err := cg.nextBlockProof(context.TODO(), base, m, ticketSets[i])
for round := int64(base.Height() + 1); len(blks) == 0; round++ {
for _, m := range miners {
proof, t, err := cg.nextBlockProof(context.TODO(), base, m, round)
if err != nil {
return nil, xerrors.Errorf("next block proof: %w", err)
}
ticketSets[i] = append(ticketSets[i], t)
if proof != nil {
fblk, err := cg.makeBlock(base, m, proof, ticketSets[i], msgs)
fblk, err := cg.makeBlock(base, m, proof, t, uint64(round), msgs)
if err != nil {
return nil, xerrors.Errorf("making a block for next tipset failed: %w", err)
}
@ -286,16 +325,16 @@ func (cg *ChainGen) NextTipSetFromMiners(base *types.TipSet, miners []address.Ad
}, nil
}
func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, eproof types.ElectionProof, tickets []*types.Ticket, msgs []*types.SignedMessage) (*types.FullBlock, error) {
func (cg *ChainGen) makeBlock(parents *types.TipSet, m address.Address, eproof *types.EPostProof, ticket *types.Ticket, height uint64, msgs []*types.SignedMessage) (*types.FullBlock, error) {
var ts uint64
if cg.Timestamper != nil {
ts = cg.Timestamper(parents, len(tickets))
ts = cg.Timestamper(parents, height-parents.Height())
} else {
ts = parents.MinTimestamp() + (uint64(len(tickets)) * build.BlockDelay)
ts = parents.MinTimestamp() + ((height - parents.Height()) * build.BlockDelay)
}
fblk, err := MinerCreateBlock(context.TODO(), cg.sm, cg.w, m, parents, tickets, eproof, msgs, ts)
fblk, err := MinerCreateBlock(context.TODO(), cg.sm, cg.w, m, parents, ticket, eproof, msgs, height, ts)
if err != nil {
return nil, err
}
@ -354,12 +393,16 @@ func (cg *ChainGen) YieldRepo() (repo.Repo, error) {
}
type MiningCheckAPI interface {
ChainGetRandomness(context.Context, types.TipSetKey, []*types.Ticket, int) ([]byte, error)
ChainGetRandomness(context.Context, types.TipSetKey, int64) ([]byte, error)
StateMinerPower(context.Context, address.Address, *types.TipSet) (api.MinerPower, error)
StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
StateMinerSectorSize(context.Context, address.Address, *types.TipSet) (uint64, error)
StateMinerProvingSet(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)
WalletSign(context.Context, address.Address, []byte) (*types.Signature, error)
}
@ -368,8 +411,8 @@ type mca struct {
sm *stmgr.StateManager
}
func (mca mca) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, ticks []*types.Ticket, lb int) ([]byte, error) {
return mca.sm.ChainStore().GetRandomness(ctx, pts.Cids(), ticks, int64(lb))
func (mca mca) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, lb int64) ([]byte, error) {
return mca.sm.ChainStore().GetRandomness(ctx, pts.Cids(), int64(lb))
}
func (mca mca) StateMinerPower(ctx context.Context, maddr address.Address, ts *types.TipSet) (api.MinerPower, error) {
@ -388,12 +431,43 @@ func (mca mca) StateMinerWorker(ctx context.Context, maddr address.Address, ts *
return stmgr.GetMinerWorkerRaw(ctx, mca.sm, ts.ParentState(), maddr)
}
func (mca mca) StateMinerSectorSize(ctx context.Context, maddr address.Address, ts *types.TipSet) (uint64, error) {
return stmgr.GetMinerSectorSize(ctx, mca.sm, ts, maddr)
}
func (mca mca) StateMinerProvingSet(ctx context.Context, maddr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
return stmgr.GetMinerProvingSet(ctx, mca.sm, ts, maddr)
}
func (mca mca) WalletSign(ctx context.Context, a address.Address, v []byte) (*types.Signature, error) {
return mca.w.Sign(ctx, a, v)
}
func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket, miner address.Address, a MiningCheckAPI) (bool, types.ElectionProof, error) {
r, err := a.ChainGetRandomness(ctx, ts.Key(), ticks, build.EcRandomnessLookback)
type ElectionPoStProver interface {
GenerateCandidates(context.Context, sectorbuilder.SortedPublicSectorInfo, []byte) ([]sectorbuilder.EPostCandidate, error)
ComputeProof(context.Context, sectorbuilder.SortedPublicSectorInfo, []byte, []sectorbuilder.EPostCandidate) ([]byte, error)
}
type eppProvider struct{}
func (epp *eppProvider) GenerateCandidates(ctx context.Context, _ sectorbuilder.SortedPublicSectorInfo, eprand []byte) ([]sectorbuilder.EPostCandidate, error) {
return []sectorbuilder.EPostCandidate{
{
SectorID: 1,
PartialTicket: [32]byte{},
Ticket: [32]byte{},
SectorChallengeIndex: 1,
},
}, nil
}
func (epp *eppProvider) ComputeProof(ctx context.Context, _ sectorbuilder.SortedPublicSectorInfo, eprand []byte, winners []sectorbuilder.EPostCandidate) ([]byte, error) {
return []byte("valid proof"), nil
}
func IsRoundWinner(ctx context.Context, ts *types.TipSet, round int64, miner address.Address, epp ElectionPoStProver, a MiningCheckAPI) (bool, *types.EPostProof, error) {
r, err := a.ChainGetRandomness(ctx, ts.Key(), round-build.EcRandomnessLookback)
if err != nil {
return false, nil, xerrors.Errorf("chain get randomness: %w", err)
}
@ -403,23 +477,131 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, ticks []*types.Ticket,
return false, nil, xerrors.Errorf("failed to get miner worker: %w", err)
}
vrfout, err := ComputeVRF(ctx, a.WalletSign, mworker, r)
vrfout, err := ComputeVRF(ctx, a.WalletSign, mworker, miner, DSepElectionPost, r)
if err != nil {
return false, nil, xerrors.Errorf("failed to compute VRF: %w", err)
}
pset, err := a.StateMinerProvingSet(ctx, miner, ts)
if err != nil {
return false, nil, xerrors.Errorf("failed to load proving set for miner: %w", err)
}
if len(pset) == 0 {
return false, nil, nil
}
var sinfos []ffi.PublicSectorInfo
for _, s := range pset {
var commRa [32]byte
copy(commRa[:], s.CommR)
sinfos = append(sinfos, ffi.PublicSectorInfo{
SectorID: s.SectorID,
CommR: commRa,
})
}
sectors := sectorbuilder.NewSortedPublicSectorInfo(sinfos)
hvrf := sha256.Sum256(vrfout)
candidates, err := epp.GenerateCandidates(ctx, sectors, hvrf[:])
if err != nil {
return false, nil, xerrors.Errorf("failed to generate electionPoSt candidates: %w", err)
}
pow, err := a.StateMinerPower(ctx, miner, ts)
if err != nil {
return false, nil, xerrors.Errorf("failed to check power: %w", err)
}
return types.PowerCmp(vrfout, pow.MinerPower, pow.TotalPower), vrfout, nil
ssize, err := a.StateMinerSectorSize(ctx, miner, ts)
if err != nil {
return false, nil, xerrors.Errorf("failed to look up miners sector size: %w", err)
}
var winners []sectorbuilder.EPostCandidate
for _, c := range candidates {
if types.IsTicketWinner(c.PartialTicket[:], ssize, pow.TotalPower) {
winners = append(winners, c)
}
}
// no winners, sad
if len(winners) == 0 {
return false, nil, nil
}
proof, err := epp.ComputeProof(ctx, sectors, hvrf[:], winners)
if err != nil {
return false, nil, xerrors.Errorf("failed to compute snark for election proof: %w", err)
}
ept := types.EPostProof{
Proof: proof,
PostRand: vrfout,
}
for _, win := range winners {
ept.Candidates = append(ept.Candidates, types.EPostTicket{
Partial: win.PartialTicket[:],
SectorID: win.SectorID,
ChallengeIndex: win.SectorChallengeIndex,
})
}
return true, &ept, nil
}
type SignFunc func(context.Context, address.Address, []byte) (*types.Signature, error)
func ComputeVRF(ctx context.Context, sign SignFunc, w address.Address, input []byte) ([]byte, error) {
sig, err := sign(ctx, w, input)
const (
DSepTicket = 1
DSepElectionPost = 2
)
func hashVRFBase(personalization uint64, miner address.Address, input []byte) ([]byte, error) {
if miner.Protocol() != address.ID {
return nil, xerrors.Errorf("miner address for compute VRF must be an ID address")
}
var persbuf [8]byte
binary.LittleEndian.PutUint64(persbuf[:], personalization)
h := sha256.New()
h.Write(persbuf[:])
h.Write([]byte{0})
h.Write(input)
h.Write([]byte{0})
h.Write(miner.Bytes())
return h.Sum(nil), nil
}
func VerifyVRF(ctx context.Context, worker, miner address.Address, p uint64, input, vrfproof []byte) error {
_, span := trace.StartSpan(ctx, "VerifyVRF")
defer span.End()
vrfBase, err := hashVRFBase(p, miner, input)
if err != nil {
return xerrors.Errorf("computing vrf base failed: %w", err)
}
sig := &types.Signature{
Type: types.KTBLS,
Data: vrfproof,
}
if err := sig.Verify(worker, vrfBase); err != nil {
return xerrors.Errorf("vrf was invalid: %w", err)
}
return nil
}
func ComputeVRF(ctx context.Context, sign SignFunc, worker, miner address.Address, p uint64, input []byte) ([]byte, error) {
sigInput, err := hashVRFBase(p, miner, input)
if err != nil {
return nil, err
}
sig, err := sign(ctx, worker, sigInput)
if err != nil {
return nil, err
}

8
chain/gen/gen_test.go
View File

@ -12,18 +12,12 @@ func testGeneration(t testing.TB, n int, msgs int) {
g.msgsPerBlock = msgs
var height int
for i := 0; i < n; i++ {
mts, err := g.NextTipSet()
if err != nil {
t.Fatalf("error at H:%d, %s", i, err)
}
ts := mts.TipSet.TipSet()
if ts.Height() != uint64(height+len(ts.Blocks()[0].Tickets)) {
t.Fatal("wrong height", ts.Height(), i, len(ts.Blocks()[0].Tickets), len(ts.Blocks()))
}
height += len(ts.Blocks()[0].Tickets)
_ = mts
}
}

12
chain/gen/mining.go
View File

@ -3,8 +3,8 @@ package gen
import (
"context"
bls "github.com/filecoin-project/filecoin-ffi"
amt "github.com/filecoin-project/go-amt-ipld"
bls "github.com/filecoin-project/go-bls-sigs"
cid "github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
cbg "github.com/whyrusleeping/cbor-gen"
@ -18,14 +18,12 @@ import (
"github.com/filecoin-project/lotus/chain/wallet"
)
func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, miner address.Address, parents *types.TipSet, tickets []*types.Ticket, proof types.ElectionProof, msgs []*types.SignedMessage, timestamp uint64) (*types.FullBlock, error) {
func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wallet, miner address.Address, parents *types.TipSet, ticket *types.Ticket, proof *types.EPostProof, msgs []*types.SignedMessage, height, timestamp uint64) (*types.FullBlock, error) {
st, recpts, err := sm.TipSetState(ctx, parents)
if err != nil {
return nil, xerrors.Errorf("failed to load tipset state: %w", err)
}
height := parents.Height() + uint64(len(tickets))
worker, err := stmgr.GetMinerWorkerRaw(ctx, sm, st, miner)
if err != nil {
return nil, xerrors.Errorf("failed to get miner worker: %w", err)
@ -34,10 +32,10 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
next := &types.BlockHeader{
Miner: miner,
Parents: parents.Cids(),
Tickets: tickets,
Ticket: ticket,
Height: height,
Timestamp: timestamp,
ElectionProof: proof,
EPostProof: *proof,
ParentStateRoot: st,
ParentMessageReceipts: recpts,
}
@ -122,7 +120,7 @@ func MinerCreateBlock(ctx context.Context, sm *stmgr.StateManager, w *wallet.Wal
return nil, xerrors.Errorf("failed to sign new block: %w", err)
}
next.BlockSig = *sig
next.BlockSig = sig
fullBlock := &types.FullBlock{
Header: next,

350
chain/gen/utils.go
View File

@ -1,6 +1,7 @@
package gen
import (
"bytes"
"context"
"fmt"
@ -8,6 +9,7 @@ import (
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
hamt "github.com/ipfs/go-hamt-ipld"
blockstore "github.com/ipfs/go-ipfs-blockstore"
bstore "github.com/ipfs/go-ipfs-blockstore"
peer "github.com/libp2p/go-libp2p-peer"
cbg "github.com/whyrusleeping/cbor-gen"
@ -20,6 +22,7 @@ import (
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/genesis"
)
type GenesisBootstrap struct {
@ -107,15 +110,6 @@ func MakeInitialStateTree(bs bstore.Blockstore, actmap map[address.Address]types
return nil, xerrors.Errorf("set storage market actor: %w", err)
}
smact, err := SetupStorageMarketActor(bs)
if err != nil {
return nil, xerrors.Errorf("setup storage market actor: %w", err)
}
if err := state.SetActor(actors.StorageMarketAddress, smact); err != nil {
return nil, xerrors.Errorf("set storage market actor: %w", err)
}
netAmt := types.FromFil(build.TotalFilecoin)
for _, amt := range actmap {
netAmt = types.BigSub(netAmt, amt)
@ -203,46 +197,61 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {
}, nil
}
func SetupStorageMarketActor(bs bstore.Blockstore) (*types.Actor, error) {
func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDeal) (cid.Cid, error) {
cst := hamt.CSTFromBstore(bs)
nd := hamt.NewNode(cst)
emptyHAMT, err := cst.Put(context.TODO(), nd)
if err != nil {
return nil, err
return cid.Undef, err
}
blks := amt.WrapBlockstore(bs)
emptyAMT, err := amt.FromArray(blks, nil)
cdeals := make([]cbg.CBORMarshaler, len(deals))
for i, deal := range deals {
cdeals[i] = &actors.OnChainDeal{
Deal: deal,
ActivationEpoch: 1,
}
}
dealAmt, err := amt.FromArray(blks, cdeals)
if err != nil {
return nil, xerrors.Errorf("amt build failed: %w", err)
return cid.Undef, xerrors.Errorf("amt build failed: %w", err)
}
sms := &actors.StorageMarketState{
Balances: emptyHAMT,
Deals: emptyAMT,
NextDealID: 0,
Deals: dealAmt,
NextDealID: uint64(len(deals)),
}
stcid, err := cst.Put(context.TODO(), sms)
if err != nil {
return nil, err
return cid.Undef, err
}
return &types.Actor{
act := &types.Actor{
Code: actors.StorageMarketCodeCid,
Head: stcid,
Nonce: 0,
Balance: types.NewInt(0),
}, nil
}
state, err := state.LoadStateTree(cst, sroot)
if err != nil {
return cid.Undef, xerrors.Errorf("making new state tree: %w", err)
}
if err := state.SetActor(actors.StorageMarketAddress, act); err != nil {
return cid.Undef, xerrors.Errorf("set storage market actor: %w", err)
}
return state.Flush()
}
type GenMinerCfg struct {
Owners []address.Address
Workers []address.Address
// not quite generating real sectors yet, but this will be necessary
//SectorDir string
PreSeals map[string]genesis.GenesisMiner
// The addresses of the created miner, this is set by the genesis setup
MinerAddrs []address.Address
@ -258,78 +267,254 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
return enc
}
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, error) {
func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDeal, error) {
vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore())
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create NewVM: %w", err)
return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err)
}
for i := 0; i < len(gmcfg.Workers); i++ {
owner := gmcfg.Owners[i]
worker := gmcfg.Workers[i]
pid := gmcfg.PeerIDs[i]
if len(gmcfg.MinerAddrs) == 0 {
return cid.Undef, nil, xerrors.New("no genesis miners")
}
params := mustEnc(&actors.CreateStorageMinerParams{
Owner: owner,
Worker: worker,
SectorSize: build.SectorSizes[0],
PeerID: pid,
})
if len(gmcfg.MinerAddrs) != len(gmcfg.PreSeals) {
return cid.Undef, nil, xerrors.Errorf("miner address list, and preseal count doesn't match (%d != %d)", len(gmcfg.MinerAddrs), len(gmcfg.PreSeals))
}
// TODO: hardcoding 7000000 here is a little fragile, it changes any
var deals []actors.StorageDeal
for i, maddr := range gmcfg.MinerAddrs {
ps, psok := gmcfg.PreSeals[maddr.String()]
if !psok {
return cid.Undef, nil, xerrors.Errorf("no preseal for miner %s", maddr)
}
minerParams := &actors.CreateStorageMinerParams{
Owner: ps.Owner,
Worker: ps.Worker,
SectorSize: ps.SectorSize,
PeerID: gmcfg.PeerIDs[i], // TODO: grab from preseal too
}
params := mustEnc(minerParams)
// TODO: hardcoding 6500 here is a little fragile, it changes any
// time anyone changes the initial account allocations
rval, err := doExecValue(ctx, vm, actors.StoragePowerAddress, owner, types.FromFil(6500), actors.SPAMethods.CreateStorageMiner, params)
rval, err := doExecValue(ctx, vm, actors.StoragePowerAddress, ps.Worker, types.FromFil(6500), actors.SPAMethods.CreateStorageMiner, params)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to create genesis miner: %w", err)
return cid.Undef, nil, xerrors.Errorf("failed to create genesis miner: %w", err)
}
maddr, err := address.NewFromBytes(rval)
maddrret, err := address.NewFromBytes(rval)
if err != nil {
return cid.Undef, err
return cid.Undef, nil, err
}
gmcfg.MinerAddrs = append(gmcfg.MinerAddrs, maddr)
_, err = vm.Flush(ctx)
if err != nil {
return cid.Undef, nil, err
}
params = mustEnc(&actors.UpdateStorageParams{Delta: types.NewInt(5000)})
cst := hamt.CSTFromBstore(cs.Blockstore())
if err := reassignMinerActorAddress(vm, cst, maddrret, maddr); err != nil {
return cid.Undef, nil, err
}
power := types.BigMul(types.NewInt(minerParams.SectorSize), types.NewInt(uint64(len(ps.Sectors))))
params = mustEnc(&actors.UpdateStorageParams{Delta: power})
_, err = doExec(ctx, vm, actors.StoragePowerAddress, maddr, actors.SPAMethods.UpdateStorage, params)
if err != nil {
return cid.Undef, xerrors.Errorf("failed to update total storage: %w", err)
return cid.Undef, nil, xerrors.Errorf("failed to update total storage: %w", err)
}
// UGLY HACKY MODIFICATION OF MINER POWER
// we have to flush the vm here because it buffers stuff internally for perf reasons
if _, err := vm.Flush(ctx); err != nil {
return cid.Undef, xerrors.Errorf("vm.Flush failed: %w", err)
return cid.Undef, nil, xerrors.Errorf("vm.Flush failed: %w", err)
}
st := vm.StateTree()
mact, err := st.GetActor(maddr)
if err != nil {
return cid.Undef, xerrors.Errorf("get miner actor failed: %w", err)
return cid.Undef, nil, xerrors.Errorf("get miner actor failed: %w", err)
}
cst := hamt.CSTFromBstore(cs.Blockstore())
var mstate actors.StorageMinerActorState
if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
return cid.Undef, xerrors.Errorf("getting miner actor state failed: %w", err)
return cid.Undef, nil, xerrors.Errorf("getting miner actor state failed: %w", err)
}
mstate.Power = types.BigMul(types.NewInt(build.SectorSizes[0]), types.NewInt(uint64(len(ps.Sectors))))
blks := amt.WrapBlockstore(cs.Blockstore())
for _, s := range ps.Sectors {
nssroot, err := actors.AddToSectorSet(ctx, blks, mstate.Sectors, s.SectorID, s.CommR[:], s.CommD[:])
if err != nil {
return cid.Undef, nil, xerrors.Errorf("failed to add fake sector to sector set: %w", err)
}
mstate.Sectors = nssroot
mstate.ProvingSet = nssroot
deals = append(deals, s.Deal)
}
mstate.Power = types.NewInt(5000)
nstate, err := cst.Put(ctx, &mstate)
if err != nil {
return cid.Undef, err
return cid.Undef, nil, err
}
mact.Head = nstate
if err := st.SetActor(maddr, mact); err != nil {
return cid.Undef, err
return cid.Undef, nil, err
}
// End of super haxx
}
return vm.Flush(ctx)
c, err := vm.Flush(ctx)
return c, deals, err
}
func reassignMinerActorAddress(vm *vm.VM, cst *hamt.CborIpldStore, from, to address.Address) error {
if from == to {
return nil
}
act, err := vm.StateTree().GetActor(from)
if err != nil {
return xerrors.Errorf("reassign: failed to get 'from' actor: %w", err)
}
_, err = vm.StateTree().GetActor(to)
if err == nil {
return xerrors.Errorf("cannot reassign actor, target address taken")
}
if err := vm.StateTree().SetActor(to, act); err != nil {
return xerrors.Errorf("failed to reassign actor: %w", err)
}
if err := adjustStorageMarketTracking(vm, cst, from, to); err != nil {
return xerrors.Errorf("adjusting storage market tracking: %w", err)
}
// Now, adjust the tracking in the init actor
return initActorReassign(vm, cst, from, to)
}
func adjustStorageMarketTracking(vm *vm.VM, cst *hamt.CborIpldStore, from, to address.Address) error {
ctx := context.TODO()
act, err := vm.StateTree().GetActor(actors.StoragePowerAddress)
if err != nil {
return xerrors.Errorf("loading storage power actor: %w", err)
}
var spst actors.StoragePowerState
if err := cst.Get(ctx, act.Head, &spst); err != nil {
return xerrors.Errorf("loading storage power actor state: %w", err)
}
miners, err := hamt.LoadNode(ctx, cst, spst.Miners)
if err != nil {
return xerrors.Errorf("loading miner set: %w", err)
}
if err := miners.Delete(ctx, string(from.Bytes())); err != nil {
return xerrors.Errorf("deleting from spa set: %w", err)
}
if err := miners.Set(ctx, string(to.Bytes()), uint64(1)); err != nil {
return xerrors.Errorf("failed setting miner: %w", err)
}
if err := miners.Flush(ctx); err != nil {
return err
}
nminerscid, err := cst.Put(ctx, miners)
if err != nil {
return err
}
spst.Miners = nminerscid
nhead, err := cst.Put(ctx, &spst)
if err != nil {
return err
}
act.Head = nhead
return nil
}
func initActorReassign(vm *vm.VM, cst *hamt.CborIpldStore, from, to address.Address) error {
ctx := context.TODO()
initact, err := vm.StateTree().GetActor(actors.InitAddress)
if err != nil {
return xerrors.Errorf("couldnt get init actor: %w", err)
}
var st actors.InitActorState
if err := cst.Get(ctx, initact.Head, &st); err != nil {
return xerrors.Errorf("reassign loading init actor state: %w", err)
}
amap, err := hamt.LoadNode(ctx, cst, st.AddressMap)
if err != nil {
return xerrors.Errorf("failed to load init actor map: %w", err)
}
target, err := address.IDFromAddress(from)
if err != nil {
return xerrors.Errorf("failed to extract ID: %w", err)
}
var out string
halt := xerrors.Errorf("halt")
err = amap.ForEach(ctx, func(k string, v interface{}) error {
_, val, err := cbg.CborReadHeader(bytes.NewReader(v.(*cbg.Deferred).Raw))
if err != nil {
return xerrors.Errorf("parsing int in map failed: %w", err)
}
if val == target {
out = k
return halt
}
return nil
})
if err == nil {
return xerrors.Errorf("could not find from address in init ID map")
}
if !xerrors.Is(err, halt) {
return xerrors.Errorf("finding address in ID map failed: %w", err)
}
if err := amap.Delete(ctx, out); err != nil {
return xerrors.Errorf("deleting 'from' entry in amap: %w", err)
}
if err := amap.Set(ctx, out, target); err != nil {
return xerrors.Errorf("setting 'to' entry in amap: %w", err)
}
if err := amap.Flush(ctx); err != nil {
return xerrors.Errorf("failed to flush amap: %w", err)
}
ncid, err := cst.Put(ctx, amap)
if err != nil {
return err
}
st.AddressMap = ncid
nacthead, err := cst.Put(ctx, &st)
if err != nil {
return err
}
initact.Head = nacthead
return nil
}
func doExec(ctx context.Context, vm *vm.VM, to, from address.Address, method uint64, params []byte) ([]byte, error) {
@ -378,11 +563,21 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
// temp chainstore
cs := store.NewChainStore(bs, datastore.NewMapDatastore())
stateroot, err = SetupStorageMiners(ctx, cs, stateroot, gmcfg)
stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg)
if err != nil {
return nil, xerrors.Errorf("setup storage miners failed: %w", err)
}
stateroot, err = SetupStorageMarketActor(bs, stateroot, deals)
if err != nil {
return nil, xerrors.Errorf("setup storage market actor: %w", err)
}
stateroot, err = AdjustInitActorStartID(ctx, bs, stateroot, 1000)
if err != nil {
return nil, xerrors.Errorf("failed to adjust init actor start ID: %w", err)
}
blks := amt.WrapBlockstore(bs)
emptyroot, err := amt.FromArray(blks, nil)
@ -409,9 +604,12 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
}
b := &types.BlockHeader{
Miner: actors.InitAddress,
Tickets: []*types.Ticket{genesisticket},
ElectionProof: []byte("the Genesis block"),
Miner: actors.InitAddress,
Ticket: genesisticket,
EPostProof: types.EPostProof{
Proof: []byte("not a real proof"),
PostRand: []byte("i guess this is kinda random"),
},
Parents: []cid.Cid{},
Height: 0,
ParentWeight: types.NewInt(0),
@ -419,7 +617,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
Messages: mmb.Cid(),
ParentMessageReceipts: emptyroot,
BLSAggregate: types.Signature{Type: types.KTBLS, Data: []byte("signatureeee")},
BlockSig: types.Signature{Type: types.KTBLS, Data: []byte("block signatureeee")},
BlockSig: &types.Signature{Type: types.KTBLS, Data: []byte("block signatureeee")},
Timestamp: ts,
}
@ -436,3 +634,37 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
Genesis: b,
}, nil
}
func AdjustInitActorStartID(ctx context.Context, bs blockstore.Blockstore, stateroot cid.Cid, val uint64) (cid.Cid, error) {
cst := hamt.CSTFromBstore(bs)
tree, err := state.LoadStateTree(cst, stateroot)
if err != nil {
return cid.Undef, err
}
act, err := tree.GetActor(actors.InitAddress)
if err != nil {
return cid.Undef, err
}
var st actors.InitActorState
if err := cst.Get(ctx, act.Head, &st); err != nil {
return cid.Undef, err
}
st.NextID = val
nstate, err := cst.Put(ctx, &st)
if err != nil {
return cid.Undef, err
}
act.Head = nstate
if err := tree.SetActor(actors.InitAddress, act); err != nil {
return cid.Undef, err
}
return tree.Flush()
}

2
chain/stmgr/call.go
View File

@ -68,7 +68,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
state := ts.ParentState()
r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height(), nil)
r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
return sm.CallRaw(ctx, msg, state, r, ts.Height())
}

28
chain/stmgr/stmgr.go
View File

@ -15,7 +15,7 @@ import (
cbg "github.com/whyrusleeping/cbor-gen"
"golang.org/x/xerrors"
bls "github.com/filecoin-project/go-bls-sigs"
bls "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
logging "github.com/ipfs/go-log"
@ -102,7 +102,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
cids[i] = v.Cid()
}
r := store.NewChainRand(sm.cs, cids, blks[0].Height, nil)
r := store.NewChainRand(sm.cs, cids, blks[0].Height)
vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore())
if err != nil {
@ -113,9 +113,14 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err)
}
reward := vm.MiningReward(netact.Balance)
for _, b := range blks {
netact, err = vmi.StateTree().GetActor(actors.NetworkAddress)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err)
}
vmi.SetBlockMiner(b.Miner)
owner, err := GetMinerOwner(ctx, sm, pstate, b.Miner)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("failed to get owner for miner %s: %w", b.Miner, err)
@ -130,6 +135,23 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
return cid.Undef, cid.Undef, xerrors.Errorf("failed to deduct funds from network actor: %w", err)
}
// all block miners created a valid post, go update the actor state
postSubmitMsg := &types.Message{
From: actors.NetworkAddress,
Nonce: netact.Nonce,
To: b.Miner,
Method: actors.MAMethods.SubmitElectionPoSt,
GasPrice: types.NewInt(0),
GasLimit: types.NewInt(10000000000),
Value: types.NewInt(0),
}
ret, err := vmi.ApplyMessage(ctx, postSubmitMsg)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message invocation failed: %w", err)
}
if ret.ExitCode != 0 {
return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d", ret.ExitCode)
}
}
// TODO: can't use method from chainstore because it doesnt let us know who the block miners were

37
chain/stmgr/utils.go
View File

@ -3,10 +3,13 @@ package stmgr
import (
"context"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
amt "github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"
@ -147,21 +150,21 @@ func GetMinerWorker(ctx context.Context, sm *StateManager, ts *types.TipSet, mad
return address.NewFromBytes(recp.Return)
}
func GetMinerProvingPeriodEnd(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) {
func GetMinerElectionPeriodStart(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return 0, xerrors.Errorf("failed to load miner actor state: %w", err)
return 0, xerrors.Errorf("(get eps) failed to load miner actor state: %w", err)
}
return mas.ProvingPeriodEnd, nil
return mas.ElectionPeriodStart, nil
}
func GetMinerProvingSet(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) ([]*api.ChainSectorInfo, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
return nil, xerrors.Errorf("(get pset) failed to load miner actor state: %w", err)
}
return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.ProvingSet)
@ -171,17 +174,37 @@ func GetMinerSectorSet(ctx context.Context, sm *StateManager, ts *types.TipSet,
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return nil, xerrors.Errorf("failed to load miner actor state: %w", err)
return nil, xerrors.Errorf("(get sset) failed to load miner actor state: %w", err)
}
return LoadSectorsFromSet(ctx, sm.ChainStore().Blockstore(), mas.Sectors)
}
func GetSectorsForElectionPost(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (*sectorbuilder.SortedPublicSectorInfo, error) {
sectors, err := GetMinerProvingSet(ctx, sm, ts, maddr)
if err != nil {
return nil, xerrors.Errorf("failed to get sector set for miner: %w", err)
}
var uselessOtherArray []ffi.PublicSectorInfo
for _, s := range sectors {
var uselessBuffer [32]byte
copy(uselessBuffer[:], s.CommR)
uselessOtherArray = append(uselessOtherArray, ffi.PublicSectorInfo{
SectorID: s.SectorID,
CommR: uselessBuffer,
})
}
ssi := sectorbuilder.NewSortedPublicSectorInfo(uselessOtherArray)
return &ssi, nil
}
func GetMinerSectorSize(ctx context.Context, sm *StateManager, ts *types.TipSet, maddr address.Address) (uint64, error) {
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return 0, xerrors.Errorf("failed to load miner actor state: %w", err)
return 0, xerrors.Errorf("(get ssize) failed to load miner actor state: %w", err)
}
cst := hamt.CSTFromBstore(sm.cs.Blockstore())
@ -197,7 +220,7 @@ func GetMinerSlashed(ctx context.Context, sm *StateManager, ts *types.TipSet, ma
var mas actors.StorageMinerActorState
_, err := sm.LoadActorState(ctx, maddr, &mas, ts)
if err != nil {
return 0, xerrors.Errorf("failed to load miner actor state: %w", err)
return 0, xerrors.Errorf("(get mslash) failed to load miner actor state: %w", err)
}
return mas.SlashedAt, nil

86
chain/store/store.go
View File

@ -3,8 +3,8 @@ package store
import (
"context"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"fmt"
"sync"
"github.com/filecoin-project/lotus/build"
@ -13,7 +13,6 @@ import (
"github.com/filecoin-project/lotus/chain/vm"
"go.opencensus.io/trace"
"go.uber.org/multierr"
"go.uber.org/zap"
amt "github.com/filecoin-project/go-amt-ipld"
"github.com/filecoin-project/lotus/chain/types"
@ -789,24 +788,21 @@ func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) {
return NewFullTipSet(out), nil
}
func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, tickets []*types.Ticket, lb int64) ([]byte, error) {
ctx, span := trace.StartSpan(ctx, "store.GetRandomness")
func drawRandomness(t *types.Ticket, round int64) []byte {
h := sha256.New()
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], uint64(round))
h.Write(t.VRFProof)
h.Write(buf[:])
return h.Sum(nil)
}
func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, round int64) ([]byte, error) {
_, span := trace.StartSpan(ctx, "store.GetRandomness")
defer span.End()
span.AddAttributes(trace.Int64Attribute("lb", lb))
if lb < 0 {
return nil, fmt.Errorf("negative lookback parameters are not valid (got %d)", lb)
}
lt := int64(len(tickets))
if lb < lt {
log.Desugar().Warn("self sampling randomness. this should be extremely rare, if you see this often it may be a bug", zap.Stack("stacktrace"))
t := tickets[lt-(1+lb)]
return t.VRFProof, nil
}
nv := lb - lt
span.AddAttributes(trace.Int64Attribute("round", round))
for {
nts, err := cs.LoadTipSet(blks)
@ -815,26 +811,21 @@ func (cs *ChainStore) GetRandomness(ctx context.Context, blks []cid.Cid, tickets
}
mtb := nts.MinTicketBlock()
lt := int64(len(mtb.Tickets))
if nv < lt {
t := mtb.Tickets[lt-(1+nv)]
return t.VRFProof, nil
}
nv -= lt
if int64(nts.Height()) <= round {
return drawRandomness(nts.MinTicketBlock().Ticket, round), nil
}
// special case for lookback behind genesis block
// TODO(spec): this is not in the spec, need to sync that
if mtb.Height == 0 {
t := mtb.Tickets[0]
// round is negative
thash := drawRandomness(mtb.Ticket, round*-1)
rval := t.VRFProof
for i := int64(0); i < nv; i++ {
h := sha256.Sum256(rval)
rval = h[:]
}
return rval, nil
// for negative lookbacks, just use the hash of the positive tickethash value
h := sha256.Sum256(thash)
return h[:], nil
}
blks = mtb.Parents
@ -855,36 +846,33 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h uint64, ts *types
}
for {
mtb := ts.MinTicketBlock()
if h >= ts.Height()-uint64(len(mtb.Tickets)) {
return ts, nil
}
pts, err := cs.LoadTipSet(ts.Parents())
if err != nil {
return nil, err
}
if h > pts.Height() {
return ts, nil
}
ts = pts
}
}
type chainRand struct {
cs *ChainStore
blks []cid.Cid
bh uint64
tickets []*types.Ticket
cs *ChainStore
blks []cid.Cid
bh uint64
}
func NewChainRand(cs *ChainStore, blks []cid.Cid, bheight uint64, tickets []*types.Ticket) vm.Rand {
func NewChainRand(cs *ChainStore, blks []cid.Cid, bheight uint64) vm.Rand {
return &chainRand{
cs: cs,
blks: blks,
bh: bheight,
tickets: tickets,
cs: cs,
blks: blks,
bh: bheight,
}
}
func (cr *chainRand) GetRandomness(ctx context.Context, h int64) ([]byte, error) {
lb := (int64(cr.bh) + int64(len(cr.tickets))) - h
return cr.cs.GetRandomness(ctx, cr.blks, cr.tickets, lb)
func (cr *chainRand) GetRandomness(ctx context.Context, round int64) ([]byte, error) {
return cr.cs.GetRandomness(ctx, cr.blks, round)
}

2
chain/store/weight.go
View File

@ -58,7 +58,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
bstate := ts.ParentState()
r := NewChainRand(cs, ts.Cids(), ts.Height(), nil)
r := NewChainRand(cs, ts.Cids(), ts.Height())
vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs)
if err != nil {

136
chain/sync.go
View File

@ -3,14 +3,15 @@ package chain
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"sync"
"time"
"github.com/Gurpartap/async"
bls "github.com/filecoin-project/filecoin-ffi"
amt "github.com/filecoin-project/go-amt-ipld"
"github.com/filecoin-project/go-bls-sigs"
"github.com/hashicorp/go-multierror"
"github.com/ipfs/go-cid"
dstore "github.com/ipfs/go-datastore"
@ -28,10 +29,12 @@ import (
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("chain")
@ -506,33 +509,8 @@ func (syncer *Syncer) minerIsValid(ctx context.Context, maddr address.Address, b
return nil
}
func (syncer *Syncer) validateTickets(ctx context.Context, mworker address.Address, tickets []*types.Ticket, base *types.TipSet) error {
ctx, span := trace.StartSpan(ctx, "validateTickets")
defer span.End()
span.AddAttributes(trace.Int64Attribute("tickets", int64(len(tickets))))
if len(tickets) == 0 {
return xerrors.Errorf("block had no tickets")
}
cur := base.MinTicket()
for i := 0; i < len(tickets); i++ {
next := tickets[i]
sig := &types.Signature{
Type: types.KTBLS,
Data: next.VRFProof,
}
// TODO: ticket signatures should also include miner address
if err := sig.Verify(mworker, cur.VRFProof); err != nil {
return xerrors.Errorf("invalid ticket, VRFProof invalid: %w", err)
}
cur = next
}
return nil
func (syncer *Syncer) validateTicket(ctx context.Context, maddr, mworker address.Address, ticket *types.Ticket, base *types.TipSet) error {
return gen.VerifyVRF(ctx, mworker, maddr, gen.DSepTicket, base.MinTicket().VRFProof, ticket.VRFProof)
}
var ErrTemporal = errors.New("temporal error")
@ -541,6 +519,9 @@ var ErrTemporal = errors.New("temporal error")
func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) error {
ctx, span := trace.StartSpan(ctx, "validateBlock")
defer span.End()
if build.InsecurePoStValidation {
log.Warn("insecure test validation is enabled, if you see this outside of a test, it is a severe bug!")
}
h := b.Header
@ -550,23 +531,34 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
}
// fast checks first
if h.BlockSig == nil {
return xerrors.Errorf("block had nil signature")
}
if h.Timestamp > uint64(time.Now().Unix()+build.AllowableClockDrift) {
return xerrors.Errorf("block was from the future")
}
if h.Timestamp < baseTs.MinTimestamp()+uint64(build.BlockDelay*len(h.Tickets)) {
log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), len(h.Tickets))
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * tkts.len:%d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, len(h.Tickets))
if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelay*(h.Height-baseTs.Height())) {
log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height())
return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, h.Height-baseTs.Height())
}
winnerCheck := async.Err(func() error {
mpow, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner)
_, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner)
if err != nil {
return xerrors.Errorf("failed getting power: %w", err)
}
if !types.PowerCmp(h.ElectionProof, mpow, tpow) {
return xerrors.Errorf("miner created a block but was not a winner")
ssize, err := stmgr.GetMinerSectorSize(ctx, syncer.sm, baseTs, h.Miner)
if err != nil {
return xerrors.Errorf("failed to get sector size for block miner: %w", err)
}
for _, t := range h.EPostProof.Candidates {
if !types.IsTicketWinner(t.Partial, ssize, tpow) {
return xerrors.Errorf("miner created a block but was not a winner")
}
}
return nil
})
@ -623,20 +615,19 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
})
tktsCheck := async.Err(func() error {
if err := syncer.validateTickets(ctx, waddr, h.Tickets, baseTs); err != nil {
vrfBase := baseTs.MinTicket().VRFProof
err := gen.VerifyVRF(ctx, waddr, h.Miner, gen.DSepTicket, vrfBase, h.Ticket.VRFProof)
if err != nil {
return xerrors.Errorf("validating block tickets failed: %w", err)
}
return nil
})
eproofCheck := async.Err(func() error {
rand, err := syncer.sm.ChainStore().GetRandomness(ctx, baseTs.Cids(), h.Tickets, build.EcRandomnessLookback)
if err != nil {
return xerrors.Errorf("failed to get randomness for verifying election proof: %w", err)
}
if err := VerifyElectionProof(ctx, h.ElectionProof, rand, waddr); err != nil {
return xerrors.Errorf("checking eproof failed: %w", err)
if err := syncer.VerifyElectionPoStProof(ctx, h, baseTs, waddr); err != nil {
return xerrors.Errorf("invalid election post: %w", err)
}
return nil
})
@ -660,6 +651,56 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
return merr
}
func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.BlockHeader, baseTs *types.TipSet, waddr address.Address) error {
rand, err := syncer.sm.ChainStore().GetRandomness(ctx, baseTs.Cids(), int64(h.Height-build.EcRandomnessLookback))
if err != nil {
return xerrors.Errorf("failed to get randomness for verifying election proof: %w", err)
}
if err := VerifyElectionPoStVRF(ctx, h.EPostProof.PostRand, rand, waddr, h.Miner); err != nil {
return xerrors.Errorf("checking eproof failed: %w", err)
}
ssize, err := stmgr.GetMinerSectorSize(ctx, syncer.sm, baseTs, h.Miner)
if err != nil {
return xerrors.Errorf("failed to get sector size for miner: %w", err)
}
var winners []sectorbuilder.EPostCandidate
for _, t := range h.EPostProof.Candidates {
var partial [32]byte
copy(partial[:], t.Partial)
winners = append(winners, sectorbuilder.EPostCandidate{
PartialTicket: partial,
SectorID: t.SectorID,
SectorChallengeIndex: t.ChallengeIndex,
})
}
sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner)
if err != nil {
return xerrors.Errorf("getting election post sector set: %w", err)
}
if build.InsecurePoStValidation {
if string(h.EPostProof.Proof) == "valid proof" {
return nil
}
return xerrors.Errorf("[TESTING] election post was invalid")
}
hvrf := sha256.Sum256(h.EPostProof.PostRand)
ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
if err != nil {
return xerrors.Errorf("failed to verify election post: %w", err)
}
if !ok {
return xerrors.Errorf("election post was invalid")
}
return nil
}
func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock, baseTs *types.TipSet) error {
nonces := make(map[address.Address]uint64)
balances := make(map[address.Address]types.BigInt)
@ -1103,14 +1144,9 @@ func (syncer *Syncer) collectChain(ctx context.Context, ts *types.TipSet) error
return nil
}
func VerifyElectionProof(ctx context.Context, eproof []byte, rand []byte, worker address.Address) error {
sig := types.Signature{
Data: eproof,
Type: types.KTBLS,
}
if err := sig.Verify(worker, rand); err != nil {
return xerrors.Errorf("failed to verify election proof signature: %w", err)
func VerifyElectionPoStVRF(ctx context.Context, evrf []byte, rand []byte, worker, miner address.Address) error {
if err := gen.VerifyVRF(ctx, worker, miner, gen.DSepElectionPost, rand, evrf); err != nil {
return xerrors.Errorf("failed to verify post_randomness vrf: %w", err)
}
return nil

8
chain/sync_test.go
View File

@ -3,6 +3,7 @@ package chain_test
import (
"context"
"fmt"
"os"
"testing"
"time"
@ -23,6 +24,11 @@ import (
"github.com/filecoin-project/lotus/node/repo"
)
func init() {
build.InsecurePoStValidation = true
os.Setenv("TRUST_PARAMS", "1")
}
const source = 0
func (tu *syncTestUtil) repoWithChain(t testing.TB, h int) (repo.Repo, []byte, []*store.FullTipSet) {
@ -390,7 +396,7 @@ func TestSyncBadTimestamp(t *testing.T) {
tu.waitUntilSync(0, client)
base := tu.g.CurTipset
tu.g.Timestamper = func(pts *types.TipSet, tl int) uint64 {
tu.g.Timestamper = func(pts *types.TipSet, tl uint64) uint64 {
return pts.MinTimestamp() + (build.BlockDelay / 2)
}

46
chain/types/blockheader.go
View File

@ -20,14 +20,24 @@ type Ticket struct {
VRFProof []byte
}
type ElectionProof []byte
type EPostTicket struct {
Partial []byte
SectorID uint64
ChallengeIndex uint64
}
type EPostProof struct {
Proof []byte
PostRand []byte
Candidates []EPostTicket
}
type BlockHeader struct {
Miner address.Address
Tickets []*Ticket
Ticket *Ticket
ElectionProof []byte
EPostProof EPostProof
Parents []cid.Cid
@ -45,7 +55,7 @@ type BlockHeader struct {
Timestamp uint64
BlockSig Signature
BlockSig *Signature
}
func (b *BlockHeader) ToStorageBlock() (block.Block, error) {
@ -91,12 +101,12 @@ func (blk *BlockHeader) Serialize() ([]byte, error) {
}
func (blk *BlockHeader) LastTicket() *Ticket {
return blk.Tickets[len(blk.Tickets)-1]
return blk.Ticket
}
func (blk *BlockHeader) SigningBytes() ([]byte, error) {
blkcopy := *blk
blkcopy.BlockSig = Signature{}
blkcopy.BlockSig = nil
return blkcopy.Serialize()
}
@ -162,30 +172,34 @@ func CidArrsEqual(a, b []cid.Cid) bool {
var blocksPerEpoch = NewInt(build.BlocksPerEpoch)
func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool {
const sha256bits = 256
func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool {
ssize := NewInt(ssizeI)
/*
Need to check that
(h(vrfout) + 1) / (max(h) + 1) <= e * minerPower / totalPower
(h(vrfout) + 1) / (max(h) + 1) <= e * sectorSize / totalPower
max(h) == 2^256-1
which in terms of integer math means:
(h(vrfout) + 1) * totalPower <= e * minerPower * 2^256
(h(vrfout) + 1) * totalPower <= e * sectorSize * 2^256
in 2^256 space, it is equivalent to:
h(vrfout) * totalPower < e * minerPower * 2^256
h(vrfout) * totalPower < e * sectorSize * 2^256
*/
h := sha256.Sum256(eproof)
h := sha256.Sum256(partialTicket)
lhs := BigFromBytes(h[:]).Int
lhs = lhs.Mul(lhs, totpow.Int)
// rhs = minerPower * 2^256
// rhs = minerPower << 256
rhs := new(big.Int).Lsh(mpow.Int, 256)
// rhs = sectorSize * 2^256
// rhs = sectorSize << 256
rhs := new(big.Int).Lsh(ssize.Int, sha256bits)
rhs = rhs.Mul(rhs, big.NewInt(build.SectorChallengeRatioDiv))
rhs = rhs.Mul(rhs, blocksPerEpoch.Int)
// h(vrfout) * totalPower < e * minerPower * 2^256?
return lhs.Cmp(rhs) == -1
// h(vrfout) * totalPower < e * sectorSize * 2^256?
return lhs.Cmp(rhs) < 0
}
func (t *Ticket) Equals(ot *Ticket) bool {

18
chain/types/blockheader_test.go
View File

@ -2,6 +2,7 @@ package types
import (
"bytes"
"fmt"
"reflect"
"testing"
@ -23,12 +24,13 @@ func testBlockHeader(t testing.TB) *BlockHeader {
}
return &BlockHeader{
Miner: addr,
ElectionProof: []byte("cats won the election"),
Tickets: []*Ticket{
&Ticket{
VRFProof: []byte("vrf proof0000000vrf proof0000000"),
},
Miner: addr,
EPostProof: EPostProof{
Proof: []byte("pruuf"),
PostRand: []byte("random"),
},
Ticket: &Ticket{
VRFProof: []byte("vrf proof0000000vrf proof0000000"),
},
Parents: []cid.Cid{c, c},
ParentMessageReceipts: c,
@ -37,7 +39,7 @@ func testBlockHeader(t testing.TB) *BlockHeader {
Messages: c,
Height: 85919298723,
ParentStateRoot: c,
BlockSig: Signature{Type: KTBLS, Data: []byte("boo! im a signature")},
BlockSig: &Signature{Type: KTBLS, Data: []byte("boo! im a signature")},
}
}
@ -55,6 +57,8 @@ func TestBlockHeaderSerialization(t *testing.T) {
}
if !reflect.DeepEqual(&out, bh) {
fmt.Printf("%#v\n", &out)
fmt.Printf("%#v\n", bh)
t.Fatal("not equal")
}
}

288
chain/types/cbor_gen.go
View File

@ -28,21 +28,13 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
return err
}
// t.t.Tickets ([]*types.Ticket) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Tickets)))); err != nil {
// t.t.Ticket (types.Ticket) (struct)
if err := t.Ticket.MarshalCBOR(w); err != nil {
return err
}
for _, v := range t.Tickets {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
// t.t.ElectionProof ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.ElectionProof)))); err != nil {
return err
}
if _, err := w.Write(t.ElectionProof); err != nil {
// t.t.EPostProof (types.EPostProof) (struct)
if err := t.EPostProof.MarshalCBOR(w); err != nil {
return err
}
@ -125,48 +117,35 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
}
}
// t.t.Tickets ([]*types.Ticket) (slice)
// t.t.Ticket (types.Ticket) (struct)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Tickets: array too large (%d)", extra)
}
{
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Tickets = make([]*Ticket, extra)
}
for i := 0; i < int(extra); i++ {
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.Ticket = new(Ticket)
if err := t.Ticket.UnmarshalCBOR(br); err != nil {
return err
}
}
var v Ticket
if err := v.UnmarshalCBOR(br); err != nil {
}
// t.t.EPostProof (types.EPostProof) (struct)
{
if err := t.EPostProof.UnmarshalCBOR(br); err != nil {
return err
}
t.Tickets[i] = &v
}
// t.t.ElectionProof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.ElectionProof: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.ElectionProof = make([]byte, extra)
if _, err := io.ReadFull(br, t.ElectionProof); err != nil {
return err
}
// t.t.Parents ([]cid.Cid) (slice)
@ -271,9 +250,21 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
{
if err := t.BlockSig.UnmarshalCBOR(br); err != nil {
pb, err := br.PeekByte()
if err != nil {
return err
}
if pb == cbg.CborNull[0] {
var nbuf [1]byte
if _, err := br.Read(nbuf[:]); err != nil {
return err
}
} else {
t.BlockSig = new(Signature)
if err := t.BlockSig.UnmarshalCBOR(br); err != nil {
return err
}
}
}
return nil
@ -333,6 +324,205 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error {
return nil
}
func (t *EPostProof) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.t.Proof ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil {
return err
}
if _, err := w.Write(t.Proof); err != nil {
return err
}
// t.t.PostRand ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PostRand)))); err != nil {
return err
}
if _, err := w.Write(t.PostRand); err != nil {
return err
}
// t.t.Candidates ([]types.EPostTicket) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Candidates)))); err != nil {
return err
}
for _, v := range t.Candidates {
if err := v.MarshalCBOR(w); err != nil {
return err
}
}
return nil
}
func (t *EPostProof) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Proof ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Proof: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Proof = make([]byte, extra)
if _, err := io.ReadFull(br, t.Proof); err != nil {
return err
}
// t.t.PostRand ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.PostRand: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.PostRand = make([]byte, extra)
if _, err := io.ReadFull(br, t.PostRand); err != nil {
return err
}
// t.t.Candidates ([]types.EPostTicket) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Candidates: array too large (%d)", extra)
}
if maj != cbg.MajArray {
return fmt.Errorf("expected cbor array")
}
if extra > 0 {
t.Candidates = make([]EPostTicket, extra)
}
for i := 0; i < int(extra); i++ {
var v EPostTicket
if err := v.UnmarshalCBOR(br); err != nil {
return err
}
t.Candidates[i] = v
}
return nil
}
func (t *EPostTicket) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{131}); err != nil {
return err
}
// t.t.Partial ([]uint8) (slice)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Partial)))); err != nil {
return err
}
if _, err := w.Write(t.Partial); err != nil {
return err
}
// t.t.SectorID (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
return err
}
// t.t.ChallengeIndex (uint64) (uint64)
if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ChallengeIndex))); err != nil {
return err
}
return nil
}
func (t *EPostTicket) UnmarshalCBOR(r io.Reader) error {
br := cbg.GetPeeker(r)
maj, extra, err := cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajArray {
return fmt.Errorf("cbor input should be of type array")
}
if extra != 3 {
return fmt.Errorf("cbor input had wrong number of fields")
}
// t.t.Partial ([]uint8) (slice)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if extra > 8192 {
return fmt.Errorf("t.Partial: array too large (%d)", extra)
}
if maj != cbg.MajByteString {
return fmt.Errorf("expected byte array")
}
t.Partial = make([]byte, extra)
if _, err := io.ReadFull(br, t.Partial); err != nil {
return err
}
// t.t.SectorID (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.SectorID = uint64(extra)
// t.t.ChallengeIndex (uint64) (uint64)
maj, extra, err = cbg.CborReadHeader(br)
if err != nil {
return err
}
if maj != cbg.MajUnsignedInt {
return fmt.Errorf("wrong type for uint64 field")
}
t.ChallengeIndex = uint64(extra)
return nil
}
func (t *Message) MarshalCBOR(w io.Writer) error {
if t == nil {
_, err := w.Write(cbg.CborNull)

14
chain/types/mock/chain.go
View File

@ -34,12 +34,12 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types
}
return &types.BlockHeader{
Miner: addr,
ElectionProof: []byte("cats won the election"),
Tickets: []*types.Ticket{
{
VRFProof: []byte(fmt.Sprintf("====%d=====", ticketNonce)),
},
Miner: addr,
EPostProof: types.EPostProof{
Proof: []byte("election post proof proof"),
},
Ticket: &types.Ticket{
VRFProof: []byte(fmt.Sprintf("====%d=====", ticketNonce)),
},
Parents: pcids,
ParentMessageReceipts: c,
@ -48,7 +48,7 @@ func MkBlock(parents *types.TipSet, weightInc uint64, ticketNonce uint64) *types
Messages: c,
Height: height,
ParentStateRoot: c,
BlockSig: types.Signature{Type: types.KTBLS, Data: []byte("boo! im a signature")},
BlockSig: &types.Signature{Type: types.KTBLS, Data: []byte("boo! im a signature")},
}
}

2
chain/types/signature_cgo.go
View File

@ -5,7 +5,7 @@ package types
import (
"fmt"
bls "github.com/filecoin-project/go-bls-sigs"
bls "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/crypto"
"github.com/minio/blake2b-simd"

3
chain/types/tipset.go
View File

@ -165,8 +165,7 @@ func (t *Ticket) Less(o *Ticket) bool {
}
func (ts *TipSet) MinTicket() *Ticket {
b := ts.MinTicketBlock()
return b.Tickets[len(b.Tickets)-1]
return ts.MinTicketBlock().Ticket
}
func (ts *TipSet) MinTimestamp() uint64 {

10
chain/types/vmcontext.go
View File

@ -3,7 +3,7 @@ package types
import (
"context"
amt "github.com/filecoin-project/go-amt-ipld"
"github.com/filecoin-project/go-amt-ipld"
"github.com/filecoin-project/lotus/chain/actors/aerrors"
"github.com/filecoin-project/lotus/chain/address"
cid "github.com/ipfs/go-cid"
@ -48,8 +48,8 @@ type storageWrapper struct {
s Storage
}
func (sw *storageWrapper) Put(i interface{}) (cid.Cid, error) {
c, err := sw.s.Put(i.(cbg.CBORMarshaler))
func (sw *storageWrapper) Put(i cbg.CBORMarshaler) (cid.Cid, error) {
c, err := sw.s.Put(i)
if err != nil {
return cid.Undef, err
}
@ -57,8 +57,8 @@ func (sw *storageWrapper) Put(i interface{}) (cid.Cid, error) {
return c, nil
}
func (sw *storageWrapper) Get(c cid.Cid, out interface{}) error {
if err := sw.s.Get(c, out.(cbg.CBORUnmarshaler)); err != nil {
func (sw *storageWrapper) Get(c cid.Cid, out cbg.CBORUnmarshaler) error {
if err := sw.s.Get(c, out); err != nil {
return err
}

13
chain/validation/state.go
View File

@ -135,12 +135,19 @@ func (s *StateWrapper) SetSingletonActor(addr vstate.SingletonActorID, balance v
return &actorWrapper{*initact}, s.storage, s.flush(tree)
case actors.StorageMarketAddress:
smact, err := gen.SetupStorageMarketActor(s.bs)
nsroot, err := gen.SetupStorageMarketActor(s.bs, s.stateRoot, nil)
if err != nil {
return nil, nil, err
}
if err := tree.SetActor(actors.StorageMarketAddress, smact); err != nil {
return nil, nil, xerrors.Errorf("set network storage market actor: %w", err)
s.stateRoot = nsroot
tree, err = state.LoadStateTree(s.cst, s.stateRoot)
if err != nil {
return nil, nil, err
}
smact, err := tree.GetActor(actors.StorageMarketAddress)
if err != nil {
return nil, nil, err
}
return &actorWrapper{*smact}, s.storage, s.flush(tree)
case actors.StoragePowerAddress:

8
chain/vm/invoker.go
View File

@ -43,9 +43,13 @@ func newInvoker() *invoker {
func (inv *invoker) Invoke(act *types.Actor, vmctx types.VMContext, method uint64, params []byte) ([]byte, aerrors.ActorError) {
if act.Code == actors.AccountCodeCid {
return nil, aerrors.Newf(254, "cannot invoke methods on account actors")
}
code, ok := inv.builtInCode[act.Code]
if !ok {
log.Errorf("no code for actor %s", act.Code)
log.Errorf("no code for actor %s (Addr: %s)", act.Code, vmctx.Message().To)
return nil, aerrors.Newf(255, "no code for actor %s(%d)(%s)", act.Code, method, hex.EncodeToString(params))
}
if method >= uint64(len(code)) || code[method] == nil {
@ -160,7 +164,7 @@ func DumpActorState(code cid.Cid, b []byte) (interface{}, error) {
typ, ok := i.builtInState[code]
if !ok {
return nil, xerrors.New("state type for actor not found")
return nil, xerrors.Errorf("state type for actor %s not found", code)
}
rv := reflect.New(typ)

2
chain/vm/vm_test.go
View File

@ -12,7 +12,7 @@ import (
func TestBlockReward(t *testing.T) {
coffer := types.FromFil(build.MiningRewardTotal).Int
sum := new(big.Int)
N := build.HalvingPeriodBlocks
N := build.HalvingPeriodEpochs
for i := 0; i < N; i++ {
a := MiningReward(types.BigInt{coffer})
sum = sum.Add(sum, a.Int)

18
chain/wallet/wallet.go
View File

@ -7,7 +7,7 @@ import (
"strings"
"sync"
"github.com/filecoin-project/go-bls-sigs"
bls "github.com/filecoin-project/filecoin-ffi"
logging "github.com/ipfs/go-log"
"github.com/minio/blake2b-simd"
@ -41,6 +41,17 @@ func NewWallet(keystore types.KeyStore) (*Wallet, error) {
return w, nil
}
func KeyWallet(keys ...*Key) *Wallet {
m := make(map[address.Address]*Key)
for _, key := range keys {
m[key.Address] = key
}
return &Wallet{
keys: m,
}
}
func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*types.Signature, error) {
ki, err := w.findKey(addr)
if err != nil {
@ -85,6 +96,11 @@ func (w *Wallet) findKey(addr address.Address) (*Key, error) {
if ok {
return k, nil
}
if w.keystore == nil {
log.Warn("findKey didn't find the key in in-memory wallet")
return nil, nil
}
ki, err := w.keystore.Get(KNamePrefix + addr.String())
if err != nil {
if xerrors.Is(err, types.ErrKeyInfoNotFound) {

4
cli/client.go
View File

@ -234,10 +234,8 @@ var clientRetrieveCmd = &cli.Command{
fmt.Println("Failed to find file")
return nil
}
order := offers[0].Order()
order.Client = payer
if err := api.ClientRetrieve(ctx, order, cctx.Args().Get(1)); err != nil {
if err := api.ClientRetrieve(ctx, offers[0].Order(payer), cctx.Args().Get(1)); err != nil {
return err
}

1
cli/cmd.go
View File

@ -122,7 +122,6 @@ var Commands = []*cli.Command{
sendCmd,
stateCmd,
syncCmd,
unregisterMinerCmd,
versionCmd,
walletCmd,
}

32
cli/miner.go
View File

@ -1,32 +0,0 @@
package cli
import (
"fmt"
"github.com/filecoin-project/lotus/chain/address"
"gopkg.in/urfave/cli.v2"
)
var unregisterMinerCmd = &cli.Command{
Name: "unregister-miner",
Usage: "Manually unregister miner actor",
Action: func(cctx *cli.Context) error {
api, closer, err := GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := ReqContext(cctx)
if !cctx.Args().Present() {
return fmt.Errorf("must pass address of miner to unregister")
}
maddr, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}
return api.MinerUnregister(ctx, maddr)
},
}

288
cmd/lotus-bench/main.go Normal file
View File

@ -0,0 +1,288 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"time"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("lotus-bench")
type BenchResults struct {
SectorSize uint64
SealingResults []SealingResult
PostGenerateCandidates time.Duration
PostEProofCold time.Duration
PostEProofHot time.Duration
VerifyEPostCold time.Duration
VerifyEPostHot time.Duration
}
type SealingResult struct {
AddPiece time.Duration
PreCommit time.Duration
Commit time.Duration
Verify time.Duration
Unseal time.Duration
}
func main() {
logging.SetLogLevel("*", "INFO")
log.Info("Starting lotus-bench")
app := &cli.App{
Name: "lotus-bench",
Usage: "Benchmark performance of lotus on your hardware",
Version: build.Version,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "storage-dir",
Value: "~/.lotus-bench",
Usage: "Path to the storage directory that will store sectors long term",
},
&cli.Uint64Flag{
Name: "sector-size",
Value: 1024,
},
&cli.BoolFlag{
Name: "no-gpu",
Usage: "disable gpu usage for the benchmark run",
},
},
Action: func(c *cli.Context) error {
if c.Bool("no-gpu") {
os.Setenv("BELLMAN_NO_GPU", "1")
}
sdir, err := homedir.Expand(c.String("storage-dir"))
if err != nil {
return err
}
os.MkdirAll(sdir, 0775)
tsdir, err := ioutil.TempDir(sdir, "bench")
if err != nil {
return err
}
defer func() {
if err := os.RemoveAll(tsdir); err != nil {
log.Warn("remove all: ", err)
}
}()
maddr, err := address.NewFromString("t0101")
if err != nil {
return err
}
sectorSize := c.Uint64("sector-size")
mds := datastore.NewMapDatastore()
cfg := &sectorbuilder.Config{
Miner: maddr,
SectorSize: sectorSize,
WorkerThreads: 2,
CacheDir: filepath.Join(tsdir, "cache"),
SealedDir: filepath.Join(tsdir, "sealed"),
StagedDir: filepath.Join(tsdir, "staged"),
UnsealedDir: filepath.Join(tsdir, "unsealed"),
}
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
if err := os.MkdirAll(d, 0775); err != nil {
return err
}
}
if err := build.GetParams(true, false); err != nil {
return xerrors.Errorf("getting params: %w", err)
}
sb, err := sectorbuilder.New(cfg, mds)
if err != nil {
return err
}
dataSize := sectorbuilder.UserBytesForSectorSize(sectorSize)
var sealTimings []SealingResult
var sealedSectors []ffi.PublicSectorInfo
numSectors := uint64(1)
for i := uint64(1); i <= numSectors; i++ {
start := time.Now()
log.Info("Writing piece into sector...")
r := rand.New(rand.NewSource(100 + int64(i)))
pi, err := sb.AddPiece(dataSize, i, r, nil)
if err != nil {
return err
}
addpiece := time.Now()
trand := sha256.Sum256([]byte(c.String("ticket-preimage")))
ticket := sectorbuilder.SealTicket{
TicketBytes: trand,
}
log.Info("Running replication...")
pieces := []sectorbuilder.PublicPieceInfo{pi}
pco, err := sb.SealPreCommit(i, ticket, pieces)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
precommit := time.Now()
sealedSectors = append(sealedSectors, ffi.PublicSectorInfo{
CommR: pco.CommR,
SectorID: i,
})
seed := sectorbuilder.SealSeed{
BlockHeight: 101,
TicketBytes: [32]byte{1, 2, 3, 4, 5},
}
log.Info("Generating PoRep for sector")
proof, err := sb.SealCommit(i, ticket, seed, pieces, pco)
if err != nil {
return err
}
sealcommit := time.Now()
commD := pi.CommP
ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("porep proof for sector %d was invalid", i)
}
verifySeal := time.Now()
log.Info("Unsealing sector")
rc, err := sb.ReadPieceFromSealedSector(1, 0, dataSize, ticket.TicketBytes[:], commD[:])
if err != nil {
return err
}
unseal := time.Now()
if err := rc.Close(); err != nil {
return err
}
sealTimings = append(sealTimings, SealingResult{
AddPiece: addpiece.Sub(start),
PreCommit: precommit.Sub(addpiece),
Commit: sealcommit.Sub(precommit),
Verify: verifySeal.Sub(sealcommit),
Unseal: unseal.Sub(verifySeal),
})
}
beforePost := time.Now()
var challenge [32]byte
rand.Read(challenge[:])
log.Info("generating election post candidates")
sinfos := sectorbuilder.NewSortedPublicSectorInfo(sealedSectors)
candidates, err := sb.GenerateEPostCandidates(sinfos, challenge, []uint64{})
if err != nil {
return err
}
gencandidates := time.Now()
log.Info("computing election post snark (cold)")
proof1, err := sb.ComputeElectionPoSt(sinfos, challenge[:], candidates[:1])
if err != nil {
return err
}
epost1 := time.Now()
log.Info("computing election post snark (hot)")
proof2, err := sb.ComputeElectionPoSt(sinfos, challenge[:], candidates[:1])
if err != nil {
return err
}
epost2 := time.Now()
if !bytes.Equal(proof1, proof2) {
log.Warn("separate epost calls returned different proof values (this might be bad)")
}
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
if err != nil {
return err
}
if !ok {
log.Error("post verification failed")
}
verifypost1 := time.Now()
ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
if err != nil {
return err
}
if !ok {
log.Error("post verification failed")
}
verifypost2 := time.Now()
benchout := BenchResults{
SectorSize: cfg.SectorSize,
SealingResults: sealTimings,
PostGenerateCandidates: gencandidates.Sub(beforePost),
PostEProofCold: epost1.Sub(gencandidates),
PostEProofHot: epost2.Sub(epost1),
VerifyEPostCold: verifypost1.Sub(epost2),
VerifyEPostHot: verifypost2.Sub(verifypost1),
} // TODO: optionally write this as json to a file
fmt.Println("results")
fmt.Printf("seal: addPiece: %s\n", benchout.SealingResults[0].AddPiece) // TODO: average across multiple sealings
fmt.Printf("seal: preCommit: %s\n", benchout.SealingResults[0].PreCommit)
fmt.Printf("seal: Commit: %s\n", benchout.SealingResults[0].Commit)
fmt.Printf("seal: Verify: %s\n", benchout.SealingResults[0].Verify)
fmt.Printf("unseal: %s\n", benchout.SealingResults[0].Unseal)
fmt.Printf("generate candidates: %s\n", benchout.PostGenerateCandidates)
fmt.Printf("compute epost proof (cold): %s\n", benchout.PostEProofCold)
fmt.Printf("compute epost proof (hot): %s\n", benchout.PostEProofHot)
fmt.Printf("verify epost proof (cold): %s\n", benchout.VerifyEPostCold)
fmt.Printf("verify epost proof (hot): %s\n", benchout.VerifyEPostHot)
return nil
},
}
if err := app.Run(os.Args); err != nil {
log.Warn(err)
return
}
}

2
cmd/lotus-chainwatch/storage.go
View File

@ -252,7 +252,7 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
i.info.SectorSize,
i.state.Power.String(),
i.state.Active,
i.state.ProvingPeriodEnd,
i.state.ElectionPeriodStart,
i.state.SlashedAt,
); err != nil {
return err

89
cmd/lotus-seed/main.go Normal file
View File

@ -0,0 +1,89 @@
package main
import (
"os"
logging "github.com/ipfs/go-log"
"github.com/mitchellh/go-homedir"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
)
var log = logging.Logger("lotus-seed")
func main() {
logging.SetLogLevel("*", "INFO")
log.Info("Starting seed")
local := []*cli.Command{
preSealCmd,
}
app := &cli.App{
Name: "lotus-seed",
Usage: "Seal sectors for genesis miner",
Version: build.Version,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sectorbuilder-dir",
Value: "~/.genesis-sectors",
},
},
Commands: local,
}
if err := app.Run(os.Args); err != nil {
log.Warn(err)
return
}
}
var preSealCmd = &cli.Command{
Name: "pre-seal",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "miner-addr",
Value: "t0101",
Usage: "specify the future address of your miner",
},
&cli.Uint64Flag{
Name: "sector-size",
Value: build.SectorSizes[0],
Usage: "specify size of sectors to pre-seal",
},
&cli.StringFlag{
Name: "ticket-preimage",
Value: "lotus is fire",
Usage: "set the ticket preimage for sealing randomness",
},
&cli.IntFlag{
Name: "num-sectors",
Value: 1,
Usage: "select number of sectors to pre-seal",
},
},
Action: func(c *cli.Context) error {
sdir := c.String("sectorbuilder-dir")
sbroot, err := homedir.Expand(sdir)
if err != nil {
return err
}
maddr, err := address.NewFromString(c.String("miner-addr"))
if err != nil {
return err
}
gm, err := seed.PreSeal(maddr, c.Uint64("sector-size"), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")))
if err != nil {
return err
}
return seed.WriteGenesisMiner(maddr, sbroot, gm)
},
}

171
cmd/lotus-seed/seed/seed.go Normal file
View File

@ -0,0 +1,171 @@
package seed
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
var log = logging.Logger("preseal")
func PreSeal(maddr address.Address, ssize uint64, sectors int, sbroot string, preimage []byte) (*genesis.GenesisMiner, error) {
cfg := &sectorbuilder.Config{
Miner: maddr,
SectorSize: ssize,
CacheDir: filepath.Join(sbroot, "cache"),
SealedDir: filepath.Join(sbroot, "sealed"),
StagedDir: filepath.Join(sbroot, "staging"),
UnsealedDir: filepath.Join(sbroot, "unsealed"),
WorkerThreads: 2,
}
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
if err := os.MkdirAll(d, 0775); err != nil {
return nil, err
}
}
mds, err := badger.NewDatastore(filepath.Join(sbroot, "badger"), nil)
if err != nil {
return nil, err
}
if err := build.GetParams(true, false); err != nil {
return nil, xerrors.Errorf("getting params: %w", err)
}
sb, err := sectorbuilder.New(cfg, mds)
if err != nil {
return nil, err
}
r := rand.New(rand.NewSource(101))
size := sectorbuilder.UserBytesForSectorSize(ssize)
var sealedSectors []*genesis.PreSeal
for i := 0; i < sectors; i++ {
sid, err := sb.AcquireSectorId()
if err != nil {
return nil, err
}
pi, err := sb.AddPiece(size, sid, r, nil)
if err != nil {
return nil, err
}
trand := sha256.Sum256(preimage)
ticket := sectorbuilder.SealTicket{
TicketBytes: trand,
}
fmt.Println("Piece info: ", pi)
pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
if err != nil {
return nil, xerrors.Errorf("commit: %w", err)
}
log.Warn("PreCommitOutput: ", sid, pco)
sealedSectors = append(sealedSectors, &genesis.PreSeal{
CommR: pco.CommR,
CommD: pco.CommD,
SectorID: sid,
})
}
minerAddr, err := wallet.GenerateKey(types.KTBLS)
if err != nil {
return nil, err
}
miner := &genesis.GenesisMiner{
Owner: minerAddr.Address,
Worker: minerAddr.Address,
SectorSize: ssize,
Sectors: sealedSectors,
Key: minerAddr.KeyInfo,
}
if err := createDeals(miner, minerAddr, maddr, ssize); err != nil {
return nil, xerrors.Errorf("creating deals: %w", err)
}
if err := mds.Close(); err != nil {
return nil, xerrors.Errorf("closing datastore: %w", err)
}
return miner, nil
}
func WriteGenesisMiner(maddr address.Address, sbroot string, gm *genesis.GenesisMiner) error {
output := map[string]genesis.GenesisMiner{
maddr.String(): *gm,
}
out, err := json.MarshalIndent(output, "", " ")
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(sbroot, "pre-seal-"+maddr.String()+".json"), out, 0664); err != nil {
return err
}
return nil
}
func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address, ssize uint64) error {
for _, sector := range m.Sectors {
proposal := &actors.StorageDealProposal{
PieceRef: sector.CommD[:], // just one deal so this == CommP
PieceSize: sectorbuilder.UserBytesForSectorSize(ssize),
PieceSerialization: actors.SerializationUnixFSv0,
Client: k.Address,
Provider: maddr,
ProposalExpiration: 9000, // TODO: allow setting
Duration: 9000,
StoragePricePerEpoch: types.NewInt(0),
StorageCollateral: types.NewInt(0),
ProposerSignature: nil,
}
if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, proposal); err != nil {
return err
}
deal := &actors.StorageDeal{
Proposal: *proposal,
CounterSignature: nil,
}
if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, deal); err != nil {
return err
}
sector.Deal = *deal
}
return nil
}

17
cmd/lotus-storage-miner/info.go
View File

@ -60,18 +60,25 @@ var infoCmd = &cli.Command{
}
fmt.Printf("Worker use: %d / %d (+%d)\n", wstat.Total-wstat.Reserved-wstat.Free, wstat.Total, wstat.Reserved)
ppe, err := api.StateMinerProvingPeriodEnd(ctx, maddr, nil)
eps, err := api.StateMinerElectionPeriodStart(ctx, maddr, nil)
if err != nil {
return err
}
if ppe != 0 {
if eps != 0 {
head, err := api.ChainHead(ctx)
if err != nil {
return err
}
pdiff := int64(ppe - head.Height())
pdifft := pdiff * build.BlockDelay
fmt.Printf("Proving Period: %d, in %d Blocks (~%dm %ds)\n", ppe, pdiff, pdifft/60, pdifft%60)
lastEps := int64(head.Height() - eps)
lastEpsS := lastEps * build.BlockDelay
next := lastEps + build.SlashablePowerDelay
nextS := next * build.BlockDelay
fmt.Printf("PoSt Submissions:\n")
fmt.Printf("\tPrevious: Epoch %d (%d block(s), ~%dm %ds ago)\n", eps, lastEps, lastEpsS/60, lastEpsS%60)
fmt.Printf("\tDeadline: Epoch %d (in %d blocks, ~%dm %ds)\n", eps+build.SlashablePowerDelay, next, nextS/60, nextS%60)
} else {
fmt.Printf("Proving Period: Not Proving\n")
}

259
cmd/lotus-storage-miner/init.go
View File

@ -3,22 +3,36 @@ package main
import (
"context"
"crypto/rand"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/mitchellh/go-homedir"
"golang.org/x/xerrors"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/lotus/api"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/cborutil"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/storage"
)
var initCmd = &cli.Command{
@ -53,6 +67,10 @@ var initCmd = &cli.Command{
Usage: "specify sector size to use",
Value: build.SectorSizes[0],
},
&cli.StringFlag{
Name: "pre-sealed-sectors",
Usage: "specify set of presealed sectors for starting as a genesis miner",
},
},
Action: func(cctx *cli.Context) error {
log.Info("Initializing lotus storage miner")
@ -104,6 +122,59 @@ var initCmd = &cli.Command{
return err
}
if pssb := cctx.String("pre-sealed-sectors"); pssb != "" {
pssb, err := homedir.Expand(pssb)
if err != nil {
return err
}
log.Infof("moving pre-sealed-sectors from %s into newly created storage miner repo", pssb)
lr, err := r.Lock(repo.StorageMiner)
if err != nil {
return err
}
mds, err := lr.Datastore("/metadata")
if err != nil {
return err
}
oldmds, err := badger.NewDatastore(filepath.Join(pssb, "badger"), nil)
if err != nil {
return err
}
oldsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
WorkerThreads: 2,
SealedDir: filepath.Join(pssb, "sealed"),
CacheDir: filepath.Join(pssb, "cache"),
StagedDir: filepath.Join(pssb, "staging"),
UnsealedDir: filepath.Join(pssb, "unsealed"),
}, oldmds)
if err != nil {
return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err)
}
nsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
WorkerThreads: 2,
SealedDir: filepath.Join(lr.Path(), "sealed"),
CacheDir: filepath.Join(lr.Path(), "cache"),
StagedDir: filepath.Join(lr.Path(), "staging"),
UnsealedDir: filepath.Join(lr.Path(), "unsealed"),
}, mds)
if err != nil {
return xerrors.Errorf("failed to open up sectorbuilder: %w", err)
}
if err := nsb.ImportFrom(oldsb); err != nil {
return err
}
if err := lr.Close(); err != nil {
return xerrors.Errorf("unlocking repo after preseal migration: %w", err)
}
}
if err := storageMinerInit(ctx, cctx, api, r); err != nil {
log.Errorf("Failed to initialize lotus-storage-miner: %+v", err)
path, err := homedir.Expand(repoPath)
@ -124,7 +195,115 @@ var initCmd = &cli.Command{
},
}
func storageMinerInit(ctx context.Context, cctx *cli.Context, api api.FullNode, r repo.Repo) error {
func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir string, maddr address.Address, mds dtypes.MetadataDS) error {
b, err := ioutil.ReadFile(filepath.Join(presealDir, "pre-seal-"+maddr.String()+".json"))
if err != nil {
return xerrors.Errorf("reading preseal metadata: %w", err)
}
preseals := map[string]genesis.GenesisMiner{}
if err := json.Unmarshal(b, &preseals); err != nil {
return xerrors.Errorf("unmarshaling preseal metadata: %w", err)
}
meta, ok := preseals[maddr.String()]
if !ok {
return xerrors.New("got wrong preseal info")
}
for _, sector := range meta.Sectors {
sectorKey := datastore.NewKey(storage.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID))
dealID, err := findMarketDealID(ctx, api, sector.Deal)
if err != nil {
return xerrors.Errorf("finding storage deal for pre-sealed sector %d: %w", sector.SectorID, err)
}
info := &storage.SectorInfo{
State: lapi.Proving,
SectorID: sector.SectorID,
Pieces: []storage.Piece{
{
DealID: dealID,
Size: meta.SectorSize,
CommP: sector.CommD[:],
},
},
CommC: nil,
CommD: sector.CommD[:],
CommR: sector.CommR[:],
CommRLast: nil,
Proof: nil,
Ticket: storage.SealTicket{},
PreCommitMessage: nil,
Seed: storage.SealSeed{},
CommitMessage: nil,
}
b, err := cborutil.Dump(info)
if err != nil {
return err
}
if err := mds.Put(sectorKey, b); err != nil {
return err
}
proposalCid, err := sector.Deal.Proposal.Cid()
if err != nil {
return err
}
dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String())
deal := &deals.MinerDeal{
Proposal: sector.Deal.Proposal,
ProposalCid: proposalCid,
State: lapi.DealComplete,
Ref: proposalCid, // TODO: This is super wrong, but there
// are no params for CommP CIDs, we can't recover unixfs cid easily,
// and this isn't even used after the deal enters Complete state
DealID: dealID,
SectorID: sector.SectorID,
}
b, err = cborutil.Dump(deal)
if err != nil {
return err
}
if err := mds.Put(dealKey, b); err != nil {
return err
}
}
return nil
}
func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDeal) (uint64, error) {
// TODO: find a better way
// (this is only used by genesis miners)
deals, err := api.StateMarketDeals(ctx, nil)
if err != nil {
return 0, xerrors.Errorf("getting market deals: %w", err)
}
for k, v := range deals {
eq, err := cborutil.Equals(&v.Deal, &deal)
if err != nil {
return 0, err
}
if eq {
return strconv.ParseUint(k, 10, 64)
}
}
return 0, xerrors.New("deal not found")
}
func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, r repo.Repo) error {
lr, err := r.Lock(repo.StorageMiner)
if err != nil {
return err
@ -143,6 +322,11 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api api.FullNode,
return xerrors.Errorf("peer ID from private key: %w", err)
}
mds, err := lr.Datastore("/metadata")
if err != nil {
return err
}
var addr address.Address
if act := cctx.String("actor"); act != "" {
a, err := address.NewFromString(act)
@ -150,7 +334,50 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api api.FullNode,
return xerrors.Errorf("failed parsing actor flag value (%q): %w", act, err)
}
if err := configureStorageMiner(ctx, api, a, peerid, cctx.Bool("genesis-miner")); err != nil {
if cctx.Bool("genesis-miner") {
if err := mds.Put(datastore.NewKey("miner-address"), a.Bytes()); err != nil {
return err
}
sbcfg, err := modules.SectorBuilderConfig(lr.Path(), 2)(mds, api)
if err != nil {
return xerrors.Errorf("getting genesis miner sector builder config: %w", err)
}
sb, err := sectorbuilder.New(sbcfg, mds)
if err != nil {
return xerrors.Errorf("failed to set up sectorbuilder for genesis mining: %w", err)
}
epp := storage.NewElectionPoStProver(sb)
m := miner.NewMiner(api, epp)
{
if err := m.Register(a); err != nil {
return xerrors.Errorf("failed to start up genesis miner: %w", err)
}
defer func() {
if err := m.Unregister(ctx, a); err != nil {
log.Error("failed to shut down storage miner: ", err)
}
}()
if err := configureStorageMiner(ctx, api, a, peerid); err != nil {
return xerrors.Errorf("failed to configure storage miner: %w", err)
}
}
if pssb := cctx.String("pre-sealed-sectors"); pssb != "" {
log.Infof("Importing pre-sealed sector metadata for %s", a)
if err := migratePreSealMeta(ctx, api, cctx.String("pre-sealed-sectors"), a, mds); err != nil {
return xerrors.Errorf("migrating presealed sector metadata: %w", err)
}
}
return nil
}
if err := configureStorageMiner(ctx, api, a, peerid); err != nil {
return xerrors.Errorf("failed to configure storage miner: %w", err)
}
@ -165,12 +392,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api api.FullNode,
}
log.Infof("Created new storage miner: %s", addr)
ds, err := lr.Datastore("/metadata")
if err != nil {
return err
}
if err := ds.Put(datastore.NewKey("miner-address"), addr.Bytes()); err != nil {
if err := mds.Put(datastore.NewKey("miner-address"), addr.Bytes()); err != nil {
return err
}
@ -203,22 +425,7 @@ func makeHostKey(lr repo.LockedRepo) (crypto.PrivKey, error) {
return pk, nil
}
func configureStorageMiner(ctx context.Context, api api.FullNode, addr address.Address, peerid peer.ID, genmine bool) error {
if genmine {
log.Warn("Starting genesis mining. This shouldn't happen when connecting to the real network.")
// We may be one of genesis miners, start mining before trying to do any chain operations
// (otherwise our messages won't be mined)
if err := api.MinerRegister(ctx, addr); err != nil {
return err
}
defer func() {
if err := api.MinerUnregister(ctx, addr); err != nil {
log.Errorf("failed to call api.MinerUnregister: %s", err)
}
}()
}
func configureStorageMiner(ctx context.Context, api lapi.FullNode, addr address.Address, peerid peer.ID) error {
// This really just needs to be an api call at this point...
recp, err := api.StateCall(ctx, &types.Message{
To: addr,
@ -271,7 +478,7 @@ func configureStorageMiner(ctx context.Context, api api.FullNode, addr address.A
return nil
}
func createStorageMiner(ctx context.Context, api api.FullNode, peerid peer.ID, cctx *cli.Context) (addr address.Address, err error) {
func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID, cctx *cli.Context) (addr address.Address, err error) {
log.Info("Creating StorageMarket.CreateStorageMiner message")
var owner address.Address

2
cmd/lotus-storage-miner/main.go
View File

@ -70,7 +70,7 @@ func main() {
}
if err := app.Run(os.Args); err != nil {
log.Warn(err)
log.Warnf("%+v", err)
os.Exit(1)
}
}

1
cmd/lotus-storage-miner/run.go
View File

@ -33,6 +33,7 @@ var runCmd = &cli.Command{
&cli.BoolFlag{
Name: "enable-gpu-proving",
Usage: "Enable use of GPU for mining operations",
Value: true,
},
},
Action: func(cctx *cli.Context) error {

2
cmd/lotus-storage-miner/sectors.go
View File

@ -163,7 +163,7 @@ var sectorsRefsCmd = &cli.Command{
for name, refs := range refs {
fmt.Printf("Block %s:\n", name)
for _, ref := range refs {
fmt.Printf("\t%s+%d %d bytes\n", ref.Piece, ref.Offset, ref.Size)
fmt.Printf("\t%d+%d %d bytes\n", ref.SectorID, ref.Offset, ref.Size)
}
}
return nil

23
cmd/lotus/daemon.go
View File

@ -21,7 +21,8 @@ import (
)
const (
makeGenFlag = "lotus-make-random-genesis"
makeGenFlag = "lotus-make-random-genesis"
preSealedSectorsFlag = "genesis-presealed-sectors"
)
// DaemonCmd is the `go-lotus daemon` command
@ -38,6 +39,10 @@ var DaemonCmd = &cli.Command{
Value: "",
Hidden: true,
},
&cli.StringFlag{
Name: preSealedSectorsFlag,
Hidden: true,
},
&cli.StringFlag{
Name: "genesis",
Usage: "genesis file to use for first node run",
@ -51,11 +56,11 @@ var DaemonCmd = &cli.Command{
ctx := context.Background()
r, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return err
return xerrors.Errorf("opening fs repo: %w", err)
}
if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists {
return err
return xerrors.Errorf("repo init error: %w", err)
}
if err := build.GetParams(false, false); err != nil {
@ -67,9 +72,8 @@ var DaemonCmd = &cli.Command{
if cctx.String("genesis") != "" {
genBytes, err = ioutil.ReadFile(cctx.String("genesis"))
if err != nil {
return err
return xerrors.Errorf("reading genesis: %w", err)
}
}
genesis := node.Options()
@ -77,7 +81,10 @@ var DaemonCmd = &cli.Command{
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genBytes))
}
if cctx.String(makeGenFlag) != "" {
genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag)))
if cctx.String(preSealedSectorsFlag) == "" {
return xerrors.Errorf("must also pass file with miner preseal info to `--%s`", preSealedSectorsFlag)
}
genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag), cctx.String(preSealedSectorsFlag)))
}
var api api.FullNode
@ -105,12 +112,12 @@ var DaemonCmd = &cli.Command{
),
)
if err != nil {
return err
return xerrors.Errorf("initializing node: %w", err)
}
endpoint, err := r.APIEndpoint()
if err != nil {
return err
return xerrors.Errorf("getting api endpoint: %w", err)
}
// TODO: properly parse api endpoint (or make it a URL)

2
cmd/lotus/main.go
View File

@ -69,7 +69,7 @@ func main() {
Code: trace.StatusCodeFailedPrecondition,
Message: err.Error(),
})
log.Warn(err)
log.Warnf("%+v", err)
os.Exit(1)
}
return

1
extern/filecoin-ffi vendored Submodule

@ -0,0 +1 @@
Subproject commit 9faf00cb536fd86559440a09de9131520ae1ca0e

1
extern/go-bls-sigs vendored

@ -1 +0,0 @@
Subproject commit 98479d3c79620f18783da0c2c6a15f8b8eb4fa2e

1
extern/go-sectorbuilder vendored

@ -1 +0,0 @@
Subproject commit 40278d4a6623e4c81003e20444871c9362bedd61

4
gen/main.go
View File

@ -20,6 +20,8 @@ func main() {
err := gen.WriteTupleEncodersToFile("./chain/types/cbor_gen.go", "types",
types.BlockHeader{},
types.Ticket{},
types.EPostProof{},
types.EPostTicket{},
types.Message{},
types.SignedMessage{},
types.MsgMeta{},
@ -91,7 +93,7 @@ func main() {
actors.SectorPreCommitInfo{},
actors.PreCommittedSector{},
actors.MinerInfo{},
actors.SubmitPoStParams{},
actors.SubmitFallbackPoStParams{},
actors.PaymentVerifyParams{},
actors.UpdatePeerIDParams{},
actors.MultiSigActorState{},

25
genesis/types.go Normal file
View File

@ -0,0 +1,25 @@
package genesis
import (
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/types"
)
type PreSeal struct {
CommR [32]byte
CommD [32]byte
SectorID uint64
Deal actors.StorageDeal
}
type GenesisMiner struct {
Owner address.Address
Worker address.Address
SectorSize uint64
Sectors []*PreSeal
Key types.KeyInfo // TODO: separate file
}

11
go.mod
View File

@ -10,9 +10,8 @@ require (
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/filecoin-project/chain-validation v0.0.0-20191106200742-11986803c0f7
github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16
github.com/filecoin-project/go-bls-sigs v0.0.0-20190718224239-4bc4b8a7bbf8
github.com/filecoin-project/go-sectorbuilder v0.0.0-00010101000000-000000000000
github.com/filecoin-project/filecoin-ffi v0.0.0-00010101000000-000000000000
github.com/filecoin-project/go-amt-ipld v0.0.0-20191122035745-59b9dfc0efc7
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/google/go-cmp v0.3.1 // indirect
@ -41,7 +40,7 @@ require (
github.com/ipfs/go-ipfs-routing v0.1.0
github.com/ipfs/go-ipld-cbor v0.0.3
github.com/ipfs/go-ipld-format v0.0.2
github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144
github.com/ipfs/go-log v1.0.0
github.com/ipfs/go-merkledag v0.2.4
github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb
github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b
@ -108,6 +107,4 @@ require (
replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0
replace github.com/filecoin-project/go-bls-sigs => ./extern/go-bls-sigs
replace github.com/filecoin-project/go-sectorbuilder => ./extern/go-sectorbuilder
replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

20
go.sum
View File

@ -78,8 +78,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
github.com/filecoin-project/chain-validation v0.0.0-20191106200742-11986803c0f7 h1:Ags/z6ZubzKonQ9PsY9fO439yGdVg07qpdxfv/AEUno=
github.com/filecoin-project/chain-validation v0.0.0-20191106200742-11986803c0f7/go.mod h1:0/0/QUNqpF/jVzLHFncGeT3NvGPODBhGzQlNgzmoZew=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16 h1:NzojcJU1VbS6zdLG13JMYis/cQy/MrN3rxmZRq56jKA=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190919045431-3650716fff16/go.mod h1:lKjJYPg2kwbav5f78i5YA8kGccnZn18IySbpneXvaQs=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191122035745-59b9dfc0efc7 h1:lKSMm8Go6qI7+Dk3rWCNIh57wBOqVNJ21re/p7D58gc=
github.com/filecoin-project/go-amt-ipld v0.0.0-20191122035745-59b9dfc0efc7/go.mod h1:lKjJYPg2kwbav5f78i5YA8kGccnZn18IySbpneXvaQs=
github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug=
github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543/go.mod h1:mjrHv1cDGJWDlGmC0eDc1E5VJr8DmL9XMUcaFwiuKg8=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -225,8 +225,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC
github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs=
github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144 h1:5WM8S1nwquWQ3zEuNhK82NE5Di6Pd41qz9JxxvxTAIA=
github.com/ipfs/go-log v0.0.2-0.20190920042044-a609c1ae5144/go.mod h1:azGN5dH7ailfREknDDNYB0Eq4qZ/4I4Y3gO0ivjJNyM=
github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI=
github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA=
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
github.com/ipfs/go-merkledag v0.2.4 h1:ZSHQSe9BENfixUjT+MaLeHEeZGxrZQfgo3KT3SLosF8=
@ -435,9 +435,8 @@ github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNA
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@ -568,6 +567,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@ -661,9 +661,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -692,9 +691,8 @@ golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@ -737,6 +735,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -751,6 +750,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y=
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

2
lib/padreader/padreader.go
View File

@ -4,7 +4,7 @@ import (
"io"
"math/bits"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
)
func PaddedSize(size uint64) uint64 {

4
lib/sectorbuilder/files.go
View File

@ -18,6 +18,10 @@ func (sb *SectorBuilder) stagedSectorPath(sectorID uint64) string {
return filepath.Join(sb.stagedDir, sb.sectorName(sectorID))
}
func (sb *SectorBuilder) unsealedSectorPath(sectorID uint64) string {
return filepath.Join(sb.unsealedDir, sb.sectorName(sectorID))
}
func (sb *SectorBuilder) stagedSectorFile(sectorID uint64) (*os.File, error) {
return os.OpenFile(sb.stagedSectorPath(sectorID), os.O_RDWR|os.O_CREATE, 0644)
}

6
lib/sectorbuilder/mock.go
View File

@ -17,8 +17,6 @@ func TempSectorbuilder(sectorSize uint64, ds dtypes.MetadataDS) (*SectorBuilder,
sb, err := TempSectorbuilderDir(dir, sectorSize, ds)
return sb, func() {
sb.Destroy()
if err := os.RemoveAll(dir); err != nil {
log.Warn("failed to clean up temp sectorbuilder: ", err)
}
@ -31,7 +29,7 @@ func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (
return nil, err
}
metadata := filepath.Join(dir, "meta")
unsealed := filepath.Join(dir, "unsealed")
sealed := filepath.Join(dir, "sealed")
staging := filepath.Join(dir, "staging")
cache := filepath.Join(dir, "cache")
@ -41,7 +39,7 @@ func TempSectorbuilderDir(dir string, sectorSize uint64, ds dtypes.MetadataDS) (
SealedDir: sealed,
StagedDir: staging,
MetadataDir: metadata,
UnsealedDir: unsealed,
CacheDir: cache,
WorkerThreads: 2,

345
lib/sectorbuilder/sectorbuilder.go
View File

@ -5,16 +5,17 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"sync"
"unsafe"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"go.opencensus.io/trace"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
@ -26,13 +27,8 @@ var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
var log = logging.Logger("sectorbuilder")
type SectorSealingStatus = sectorbuilder.SectorSealingStatus
type StagedSectorMetadata = sectorbuilder.StagedSectorMetadata
type SortedSectorInfo = sectorbuilder.SortedSectorInfo
type SectorInfo = sectorbuilder.SectorInfo
type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo
type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo
type SealTicket = sectorbuilder.SealTicket
@ -46,20 +42,25 @@ type PublicPieceInfo = sectorbuilder.PublicPieceInfo
type RawSealPreCommitOutput = sectorbuilder.RawSealPreCommitOutput
type EPostCandidate = sectorbuilder.Candidate
const CommLen = sectorbuilder.CommitmentBytesLen
type SectorBuilder struct {
handle unsafe.Pointer
ds dtypes.MetadataDS
idLk sync.Mutex
ds dtypes.MetadataDS
idLk sync.Mutex
ssize uint64
ssize uint64
lastID uint64
Miner address.Address
stagedDir string
sealedDir string
cacheDir string
stagedDir string
sealedDir string
cacheDir string
unsealedDir string
unsealLk sync.Mutex
rateLimit chan struct{}
}
@ -73,7 +74,7 @@ type Config struct {
CacheDir string
SealedDir string
StagedDir string
MetadataDir string
UnsealedDir string
}
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
@ -81,9 +82,7 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers+1, cfg.WorkerThreads)
}
proverId := addressToProverID(cfg.Miner)
for _, dir := range []string{cfg.StagedDir, cfg.SealedDir, cfg.CacheDir, cfg.MetadataDir} {
for _, dir := range []string{cfg.StagedDir, cfg.SealedDir, cfg.CacheDir, cfg.UnsealedDir} {
if err := os.Mkdir(dir, 0755); err != nil {
if os.IsExist(err) {
continue
@ -106,20 +105,16 @@ func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
return nil, err
}
sbp, err := sectorbuilder.InitSectorBuilder(cfg.SectorSize, PoRepProofPartitions, lastUsedID, cfg.MetadataDir, proverId, cfg.SealedDir, cfg.StagedDir, cfg.CacheDir, 16, cfg.WorkerThreads)
if err != nil {
return nil, err
}
sb := &SectorBuilder{
handle: sbp,
ds: ds,
ds: ds,
ssize: cfg.SectorSize,
ssize: cfg.SectorSize,
lastID: lastUsedID,
stagedDir: cfg.StagedDir,
sealedDir: cfg.SealedDir,
cacheDir: cfg.CacheDir,
stagedDir: cfg.StagedDir,
sealedDir: cfg.SealedDir,
cacheDir: cfg.CacheDir,
unsealedDir: cfg.UnsealedDir,
Miner: cfg.Miner,
rateLimit: make(chan struct{}, cfg.WorkerThreads-PoStReservedWorkers),
@ -149,19 +144,14 @@ func addressToProverID(a address.Address) [32]byte {
return proverId
}
func (sb *SectorBuilder) Destroy() {
sectorbuilder.DestroySectorBuilder(sb.handle)
}
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
sb.idLk.Lock()
defer sb.idLk.Unlock()
id, err := sectorbuilder.AcquireSectorId(sb.handle)
if err != nil {
return 0, err
}
err = sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
sb.lastID++
id := sb.lastID
err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
if err != nil {
return 0, err
}
@ -182,7 +172,7 @@ func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Rea
return PublicPieceInfo{}, err
}
_, _, commP, err := sectorbuilder.StandaloneWriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
_, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
if err != nil {
return PublicPieceInfo{}, err
}
@ -201,12 +191,72 @@ func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Rea
}, werr()
}
// TODO: should *really really* return an io.ReadCloser
func (sb *SectorBuilder) ReadPieceFromSealedSector(pieceKey string) ([]byte, error) {
ret := sb.RateLimit()
func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker
defer ret()
return sectorbuilder.ReadPieceFromSealedSector(sb.handle, pieceKey)
sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel
defer sb.unsealLk.Unlock()
cacheDir, err := sb.sectorCacheDir(sectorID)
if err != nil {
return nil, err
}
sealedPath, err := sb.sealedSectorPath(sectorID)
if err != nil {
return nil, err
}
unsealedPath := sb.unsealedSectorPath(sectorID)
// TODO: GC for those
// (Probably configurable count of sectors to be kept unsealed, and just
// remove last used one (or use whatever other cache policy makes sense))
f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
var commd [CommLen]byte
copy(commd[:], commD)
var tkt [CommLen]byte
copy(tkt[:], ticket)
err = sectorbuilder.Unseal(sb.ssize,
PoRepProofPartitions,
cacheDir,
sealedPath,
unsealedPath,
sectorID,
addressToProverID(sb.Miner),
tkt,
commd)
if err != nil {
return nil, xerrors.Errorf("unseal failed: %w", err)
}
f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
if err != nil {
return nil, err
}
}
if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
return nil, xerrors.Errorf("seek: %w", err)
}
lr := io.LimitReader(f, int64(size))
return &struct {
io.Reader
io.Closer
}{
Reader: lr,
Closer: f,
}, nil
}
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
@ -234,7 +284,7 @@ func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, piece
stagedPath := sb.stagedSectorPath(sectorID)
rspco, err := sectorbuilder.StandaloneSealPreCommit(
rspco, err := sectorbuilder.SealPreCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
@ -252,7 +302,7 @@ func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, piece
return rspco, nil
}
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, pieceKeys []string, rspco RawSealPreCommitOutput) (proof []byte, err error) {
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
ret := sb.RateLimit()
defer ret()
@ -261,7 +311,7 @@ func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed Sea
return nil, err
}
proof, err = sectorbuilder.StandaloneSealCommit(
proof, err = sectorbuilder.SealCommit(
sb.ssize,
PoRepProofPartitions,
cacheDir,
@ -273,53 +323,86 @@ func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed Sea
rspco,
)
if err != nil {
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
return nil, xerrors.Errorf("SealCommit: %w", err)
}
pmeta := make([]sectorbuilder.PieceMetadata, len(pieces))
for i, piece := range pieces {
pmeta[i] = sectorbuilder.PieceMetadata{
Key: pieceKeys[i],
Size: piece.Size,
CommP: piece.CommP,
}
}
sealedPath, err := sb.sealedSectorPath(sectorID)
if err != nil {
return nil, err
}
err = sectorbuilder.ImportSealedSector(
sb.handle,
sectorID,
cacheDir,
sealedPath,
ticket,
seed,
rspco.CommR,
rspco.CommD,
rspco.CommC,
rspco.CommRLast,
proof,
pmeta,
)
if err != nil {
return nil, xerrors.Errorf("ImportSealedSector: %w", err)
}
return proof, nil
}
func (sb *SectorBuilder) GeneratePoSt(sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]byte, error) {
// Wait, this is a blocking method with no way of interrupting it?
// does it checkpoint itself?
return sectorbuilder.GeneratePoSt(sb.handle, sectorInfo, challengeSeed, faults)
}
func (sb *SectorBuilder) SectorSize() uint64 {
return sb.ssize
}
func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) {
if len(challengeSeed) != CommLen {
return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen)
}
var cseed [CommLen]byte
copy(cseed[:], challengeSeed)
privsects, err := sb.pubSectorToPriv(sectorInfo)
if err != nil {
return nil, err
}
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
}
func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo)
if err != nil {
return nil, err
}
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
proverID := addressToProverID(sb.Miner)
return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
}
func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo) (SortedPrivateSectorInfo, error) {
var out []sectorbuilder.PrivateSectorInfo
for _, s := range sectorInfo.Values() {
cachePath, err := sb.sectorCacheDir(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err)
}
sealedPath, err := sb.sealedSectorPath(s.SectorID)
if err != nil {
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err)
}
out = append(out, sectorbuilder.PrivateSectorInfo{
SectorID: s.SectorID,
CommR: s.CommR,
CacheDirPath: cachePath,
SealedSectorPath: sealedPath,
})
}
return NewSortedPrivateSectorInfo(out), nil
}
func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) {
privsectors, err := sb.pubSectorToPriv(sectorInfo)
if err != nil {
return nil, nil, err
}
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())))
proverID := addressToProverID(sb.Miner)
candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
if err != nil {
return nil, nil, err
}
proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates)
return candidates, proof, err
}
var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector
func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address, ticket []byte, seed []byte, sectorID uint64, proof []byte) (bool, error) {
@ -333,14 +416,37 @@ func VerifySeal(sectorSize uint64, commR, commD []byte, proverID address.Address
return sectorbuilder.VerifySeal(sectorSize, commRa, commDa, proverIDa, ticketa, seeda, sectorID, proof)
}
func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo {
return sectorbuilder.NewSortedSectorInfo(sectors...)
func NewSortedPrivateSectorInfo(sectors []sectorbuilder.PrivateSectorInfo) SortedPrivateSectorInfo {
return sectorbuilder.NewSortedPrivateSectorInfo(sectors...)
}
func VerifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, proof []byte, faults []uint64) (bool, error) {
func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedPublicSectorInfo {
return sectorbuilder.NewSortedPublicSectorInfo(sectors...)
}
func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func VerifyFallbackPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())))
return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
}
func verifyPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeCount uint64, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
if challengeCount != uint64(len(candidates)) {
log.Warnf("verifyPost with wrong candidate count: expected %d, got %d", challengeCount, len(candidates))
return false, nil // user input, dont't error
}
var challengeSeeda [CommLen]byte
copy(challengeSeeda[:], challengeSeed)
_, span := trace.StartSpan(ctx, "VerifyPoSt")
defer span.End()
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeed, proof, faults)
prover := addressToProverID(proverID)
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeeda, challengeCount, proof, candidates, prover)
}
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
@ -360,3 +466,62 @@ func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]
func GenerateDataCommitment(ssize uint64, pieces []PublicPieceInfo) ([CommLen]byte, error) {
return sectorbuilder.GenerateDataCommitment(ssize, pieces)
}
func ElectionPostChallengeCount(sectors uint64) uint64 {
// ceil(sectors / build.SectorChallengeRatioDiv)
return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv
}
func fallbackPostChallengeCount(sectors uint64) uint64 {
challengeCount := ElectionPostChallengeCount(sectors)
if challengeCount > build.MaxFallbackPostChallengeCount {
return build.MaxFallbackPostChallengeCount
}
return challengeCount
}
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder) error {
if err := moveAllFiles(osb.cacheDir, sb.cacheDir); err != nil {
return err
}
if err := moveAllFiles(osb.sealedDir, sb.sealedDir); err != nil {
return err
}
if err := moveAllFiles(osb.stagedDir, sb.stagedDir); err != nil {
return err
}
val, err := osb.ds.Get(lastSectorIdKey)
if err != nil {
return err
}
if err := sb.ds.Put(lastSectorIdKey, val); err != nil {
return err
}
sb.lastID = osb.lastID
return nil
}
func moveAllFiles(from, to string) error {
dir, err := os.Open(from)
if err != nil {
return err
}
names, err := dir.Readdirnames(0)
if err != nil {
return xerrors.Errorf("failed to list items in dir: %w", err)
}
for _, n := range names {
if err := os.Rename(filepath.Join(from, n), filepath.Join(to, n)); err != nil {
return xerrors.Errorf("moving file failed: %w", err)
}
}
return nil
}

135
lib/sectorbuilder/sectorbuilder_test.go
View File

@ -2,6 +2,7 @@ package sectorbuilder_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
@ -15,6 +16,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
)
@ -59,7 +61,7 @@ func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
}
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, []string{"foo"}, s.pco)
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco)
if err != nil {
t.Fatalf("%+v", err)
}
@ -76,34 +78,51 @@ func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()
done()
}
func (s *seal) post(t *testing.T, sb *sectorbuilder.SectorBuilder) {
func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time {
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
ssi := sectorbuilder.NewSortedSectorInfo([]sectorbuilder.SectorInfo{{
SectorID: s.sid,
CommR: s.pco.CommR,
}})
ppi := make([]ffi.PublicSectorInfo, len(seals))
for i, s := range seals {
ppi[i] = ffi.PublicSectorInfo{
SectorID: s.sid,
CommR: s.pco.CommR,
}
}
postProof, err := sb.GeneratePoSt(ssi, cSeed, []uint64{})
ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi)
candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{})
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifyPost(context.TODO(), sb.SectorSize(), ssi, cSeed, postProof, []uint64{})
genCandidates := time.Now()
if len(candndates) != 1 {
t.Fatal("expected 1 candidate")
}
postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates)
if err != nil {
t.Fatalf("%+v", err)
}
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner)
if err != nil {
t.Fatalf("%+v", err)
}
if !ok {
t.Fatal("bad post")
}
return genCandidates
}
func TestSealAndVerify(t *testing.T) {
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
os.Setenv("BELLMAN_NO_GPU", "1")
os.Setenv("RUST_LOG", "info")
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
@ -123,7 +142,10 @@ func TestSealAndVerify(t *testing.T) {
t.Fatalf("%+v", err)
}
cleanup := func() {
sb.Destroy()
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
@ -137,28 +159,101 @@ func TestSealAndVerify(t *testing.T) {
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
s.commit(t, sb, func() {})
s.post(t, sb)
commit := time.Now()
genCandidiates := post(t, sb, s)
epost := time.Now()
// Restart sectorbuilder, re-run post
sb.Destroy()
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
s.post(t, sb)
post(t, sb, s)
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("Commit: %s\n", commit.Sub(precommit).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealAndVerify2(t *testing.T) {
func TestSealPoStNoCommit(t *testing.T) {
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
os.Setenv("BELLMAN_NO_GPU", "1")
os.Setenv("RUST_LOG", "info")
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
if err := build.GetParams(true, true); err != nil {
t.Fatalf("%+v", err)
}
ds := datastore.NewMapDatastore()
dir, err := ioutil.TempDir("", "sbtest")
if err != nil {
t.Fatal(err)
}
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
cleanup := func() {
if t.Failed() {
fmt.Printf("not removing %s\n", dir)
return
}
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
}
defer cleanup()
si, err := sb.AcquireSectorId()
if err != nil {
t.Fatalf("%+v", err)
}
s := seal{sid: si}
start := time.Now()
s.precommit(t, sb, 1, func() {})
precommit := time.Now()
// Restart sectorbuilder, re-run post
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
}
genCandidiates := post(t, sb, s)
epost := time.Now()
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String())
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
}
func TestSealAndVerify2(t *testing.T) {
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
t.Skip("this is slow")
}
_ = os.Setenv("RUST_LOG", "info")
build.SectorSizes = []uint64{sectorSize}
@ -178,7 +273,6 @@ func TestSealAndVerify2(t *testing.T) {
t.Fatalf("%+v", err)
}
cleanup := func() {
sb.Destroy()
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}
@ -210,6 +304,8 @@ func TestSealAndVerify2(t *testing.T) {
go s1.commit(t, sb, wg.Done)
go s2.commit(t, sb, wg.Done)
wg.Wait()
post(t, sb, s1, s2)
}
func TestAcquireID(t *testing.T) {
@ -235,8 +331,6 @@ func TestAcquireID(t *testing.T) {
assertAcquire(2)
assertAcquire(3)
sb.Destroy()
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
if err != nil {
t.Fatalf("%+v", err)
@ -246,7 +340,6 @@ func TestAcquireID(t *testing.T) {
assertAcquire(5)
assertAcquire(6)
sb.Destroy()
if err := os.RemoveAll(dir); err != nil {
t.Error(err)
}

10
lotuspond/front/src/FullNode.js
View File

@ -45,8 +45,6 @@ class FullNode extends React.Component {
return this.props.client.call('Filecoin.PaychVoucherList', [paych])
}))
let minerList = await this.props.client.call('Filecoin.MinerAddresses', [])
let mpoolPending = (await this.props.client.call('Filecoin.MpoolPending', [tipset])).length
this.setState(() => ({
@ -62,8 +60,6 @@ class FullNode extends React.Component {
vouchers: vouchers,
defaultAddr: defaultAddr,
minerList: minerList,
}))
}
@ -110,11 +106,6 @@ class FullNode extends React.Component {
)
}
let miners = <span/>
if(this.state.minerList.length > 0) {
miners = this.state.minerList.map((a, k) => <div key={k}><Address miner={true} client={this.props.client} addr={a} mountWindow={this.props.mountWindow}/></div>)
}
let storageMine = <a href="#" onClick={this.startStorageMiner} hidden={!this.props.spawnStorageNode}>[Spawn Storage Miner]</a>
let addresses = this.state.addrs.map((addr) => {
@ -153,7 +144,6 @@ class FullNode extends React.Component {
<div>
<div>Balances: [New <a href="#" onClick={this.newSecpAddr}>[Secp256k1]</a> <a href="#" onClick={this.newBLSAddr}>[BLS]</a>]</div>
<div>{addresses}</div>
<div>{miners}</div>
<div>{paychannels}</div>
</div>

7
lotuspond/front/src/State.js
View File

@ -107,7 +107,8 @@ class MarketState extends React.Component {
const tipset = await this.props.client.call("Filecoin.ChainHead", []) // TODO: from props
const participants = await this.props.client.call("Filecoin.StateMarketParticipants", [tipset])
const deals = await this.props.client.call("Filecoin.StateMarketDeals", [tipset])
this.setState({participants, deals})
const state = await this.props.client.call('Filecoin.StateReadState', [this.props.actor, tipset])
this.setState({participants, deals, nextDeal: state.State.NextDealID})
}
render() {
@ -125,7 +126,7 @@ class MarketState extends React.Component {
</div>
<div>
<div>---</div>
<div>Deals:</div>
<div>Deals ({this.state.nextDeal} Total):</div>
<table>
<tr><td>id</td><td>Active</td><td>Client</td><td>Provider</td><td>Size</td><td>Price</td><td>Duration</td></tr>
{Object.keys(this.state.deals).map(d => <tr>
@ -181,7 +182,7 @@ class MinerState extends React.Component {
<div>Worker: <Address addr={this.state.worker} client={this.props.client} mountWindow={this.props.mountWindow}/></div>
<div>Sector Size: <b>{this.state.sectorSize/1024}</b> KiB</div>
<div>Power: <b>{state.Power}</b> (<b>{state.Power/this.state.networkPower*100}</b>%)</div>
<div>Proving Period End: <b>{state.ProvingPeriodEnd}</b></div>
<div>Election Period Start: <b>{state.ElectionPeriodStart}</b></div>
<div>Slashed: <b>{state.SlashedAt === 0 ? "NO" : state.SlashedAt}</b></div>
<div>
<div>----</div>

2
lotuspond/front/src/StorageNode.js
View File

@ -120,7 +120,7 @@ class StorageNode extends React.Component {
</div>
<div>
<Address client={this.props.fullConn} addr={this.state.actor} mountWindow={this.props.mountWindow}/>
<span>&nbsp;<abbr title="Proving period end">PPE:</abbr> <b>{this.state.actorState.State.ProvingPeriodEnd}</b></span>
<span>&nbsp;<abbr title="Proving period end">EPS:</abbr> <b>{this.state.actorState.State.ElectionPeriodStart}</b></span>
</div>
<div>{this.state.statusCounts.map((c, i) => <span key={i}>{sealCodes[i]}: {c} | </span>)}</div>
<div>

2
lotuspond/main.go
View File

@ -88,6 +88,8 @@ var shCmd = &cli.Command{
}
}
shcmd.Env = append(os.Environ(), shcmd.Env...)
shcmd.Stdin = os.Stdin
shcmd.Stdout = os.Stdout
shcmd.Stderr = os.Stderr

28
lotuspond/spawn.go
View File

@ -11,6 +11,10 @@ import (
"time"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
)
func (api *api) Spawn() (nodeInfo, error) {
@ -19,9 +23,29 @@ func (api *api) Spawn() (nodeInfo, error) {
return nodeInfo{}, err
}
params := []string{"daemon", "--bootstrap=false"}
genParam := "--genesis=" + api.genesis
id := atomic.AddInt32(&api.cmds, 1)
if id == 1 {
// preseal
genMiner, err := address.NewIDAddress(101)
if err != nil {
return nodeInfo{}, err
}
sbroot := filepath.Join(dir, "preseal")
genm, err := seed.PreSeal(genMiner, build.SectorSizes[0], 1, sbroot, []byte("8"))
if err != nil {
return nodeInfo{}, xerrors.Errorf("preseal failed: %w", err)
}
if err := seed.WriteGenesisMiner(genMiner, sbroot, genm); err != nil {
return nodeInfo{}, xerrors.Errorf("failed to write genminer info: %w", err)
}
params = append(params, "--genesis-presealed-sectors="+filepath.Join(dir, "preseal", "pre-seal-t0101.json"))
// make genesis
genf, err := ioutil.TempFile(os.TempDir(), "lotus-genesis-")
if err != nil {
@ -54,7 +78,7 @@ func (api *api) Spawn() (nodeInfo, error) {
return nodeInfo{}, err
}
cmd := exec.Command("./lotus", "daemon", "--bootstrap=false", genParam)
cmd := exec.Command("./lotus", append(params, genParam)...)
cmd.Stderr = io.MultiWriter(os.Stderr, errlogfile, mux.errpw)
cmd.Stdout = io.MultiWriter(os.Stdout, logfile, mux.outpw)
@ -114,7 +138,7 @@ func (api *api) SpawnStorage(fullNodeRepo string) (nodeInfo, error) {
initArgs := []string{"init"}
if fullNodeRepo == api.running[1].meta.Repo {
initArgs = []string{"init", "--actor=t0101", "--genesis-miner"}
initArgs = []string{"init", "--actor=t0101", "--genesis-miner", "--pre-sealed-sectors=" + filepath.Join(fullNodeRepo, "preseal")}
}
id := atomic.AddInt32(&api.cmds, 1)

89
miner/miner.go
View File

@ -5,16 +5,15 @@ import (
"sync"
"time"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/node/impl/full"
logging "github.com/ipfs/go-log"
"go.opencensus.io/trace"
"go.uber.org/fx"
"golang.org/x/xerrors"
)
@ -22,19 +21,10 @@ var log = logging.Logger("miner")
type waitFunc func(ctx context.Context) error
type api struct {
fx.In
full.ChainAPI
full.SyncAPI
full.MpoolAPI
full.WalletAPI
full.StateAPI
}
func NewMiner(api api) *Miner {
func NewMiner(api api.FullNode, epp gen.ElectionPoStProver) *Miner {
return &Miner{
api: api,
epp: epp,
waitFunc: func(ctx context.Context) error {
// Wait around for half the block time in case other parents come in
time.Sleep(build.BlockDelay * time.Second / 2)
@ -44,7 +34,9 @@ func NewMiner(api api) *Miner {
}
type Miner struct {
api api
api api.FullNode
epp gen.ElectionPoStProver
lk sync.Mutex
addresses []address.Address
@ -160,8 +152,8 @@ eventLoop:
log.Errorf("failed to get best mining candidate: %s", err)
continue
}
if base.ts.Equals(lastBase.ts) && len(lastBase.tickets) == len(base.tickets) {
log.Errorf("BestMiningCandidate from the previous round: %s (tkts:%d)", lastBase.ts.Cids(), len(lastBase.tickets))
if base.ts.Equals(lastBase.ts) && lastBase.nullRounds == base.nullRounds {
log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.ts.Cids(), lastBase.nullRounds)
time.Sleep(build.BlockDelay * time.Second)
continue
}
@ -172,7 +164,7 @@ eventLoop:
for _, addr := range addrs {
b, err := m.mineOne(ctx, addr, base)
if err != nil {
log.Errorf("mining block failed: %s", err)
log.Errorf("mining block failed: %+v", err)
continue
}
if b != nil {
@ -204,15 +196,15 @@ eventLoop:
}
}
} else {
nextRound := time.Unix(int64(base.ts.MinTimestamp()+uint64(build.BlockDelay*len(base.tickets))), 0)
nextRound := time.Unix(int64(base.ts.MinTimestamp()+uint64(build.BlockDelay*base.nullRounds)), 0)
time.Sleep(time.Until(nextRound))
}
}
}
type MiningBase struct {
ts *types.TipSet
tickets []*types.Ticket
ts *types.TipSet
nullRounds uint64
}
func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) {
@ -245,20 +237,41 @@ func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error)
}, nil
}
func (m *Miner) hasPower(ctx context.Context, addr address.Address, ts *types.TipSet) (bool, error) {
power, err := m.api.StateMinerPower(ctx, addr, ts)
if err != nil {
return false, err
}
return power.MinerPower.Equals(types.NewInt(0)), nil
}
func (m *Miner) mineOne(ctx context.Context, addr address.Address, base *MiningBase) (*types.BlockMsg, error) {
log.Debugw("attempting to mine a block", "tipset", types.LogCids(base.ts.Cids()))
ticket, err := m.scratchTicket(ctx, addr, base)
start := time.Now()
hasPower, err := m.hasPower(ctx, addr, base.ts)
if err != nil {
return nil, xerrors.Errorf("checking if miner is slashed: %w", err)
}
if hasPower {
// slashed or just have no power yet
base.nullRounds++
return nil, nil
}
ticket, err := m.computeTicket(ctx, addr, base)
if err != nil {
return nil, xerrors.Errorf("scratching ticket failed: %w", err)
}
win, proof, err := gen.IsRoundWinner(ctx, base.ts, append(base.tickets, ticket), addr, &m.api)
win, proof, err := gen.IsRoundWinner(ctx, base.ts, int64(base.ts.Height()+base.nullRounds+1), addr, m.epp, m.api)
if err != nil {
return nil, xerrors.Errorf("failed to check if we win next round: %w", err)
}
if !win {
m.submitNullTicket(base, ticket)
base.nullRounds++
return nil, nil
}
@ -268,12 +281,10 @@ func (m *Miner) mineOne(ctx context.Context, addr address.Address, base *MiningB
}
log.Infow("mined new block", "cid", b.Cid())
return b, nil
}
dur := time.Now().Sub(start)
log.Infof("Creating block took %s", dur)
func (m *Miner) submitNullTicket(base *MiningBase, ticket *types.Ticket) {
base.tickets = append(base.tickets, ticket)
m.lastWork = base
return b, nil
}
func (m *Miner) computeVRF(ctx context.Context, addr address.Address, input []byte) ([]byte, error) {
@ -282,7 +293,7 @@ func (m *Miner) computeVRF(ctx context.Context, addr address.Address, input []by
return nil, err
}
return gen.ComputeVRF(ctx, m.api.WalletSign, w, input)
return gen.ComputeVRF(ctx, m.api.WalletSign, w, addr, gen.DSepTicket, input)
}
func (m *Miner) getMinerWorker(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
@ -307,15 +318,11 @@ func (m *Miner) getMinerWorker(ctx context.Context, addr address.Address, ts *ty
return w, nil
}
func (m *Miner) scratchTicket(ctx context.Context, addr address.Address, base *MiningBase) (*types.Ticket, error) {
var lastTicket *types.Ticket
if len(base.tickets) > 0 {
lastTicket = base.tickets[len(base.tickets)-1]
} else {
lastTicket = base.ts.MinTicket()
}
func (m *Miner) computeTicket(ctx context.Context, addr address.Address, base *MiningBase) (*types.Ticket, error) {
vrfOut, err := m.computeVRF(ctx, addr, lastTicket.VRFProof)
vrfBase := base.ts.MinTicket().VRFProof
vrfOut, err := m.computeVRF(ctx, addr, vrfBase)
if err != nil {
return nil, err
}
@ -325,7 +332,7 @@ func (m *Miner) scratchTicket(ctx context.Context, addr address.Address, base *M
}, nil
}
func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *types.Ticket, proof types.ElectionProof) (*types.BlockMsg, error) {
func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *types.Ticket, proof *types.EPostProof) (*types.BlockMsg, error) {
pending, err := m.api.MpoolPending(context.TODO(), base.ts)
if err != nil {
@ -337,10 +344,12 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type
return nil, xerrors.Errorf("message filtering failed: %w", err)
}
uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(len(base.tickets)+1))
uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(base.nullRounds+1))
nheight := base.ts.Height() + base.nullRounds + 1
// why even return this? that api call could just submit it for us
return m.api.MinerCreateBlock(context.TODO(), addr, base.ts, append(base.tickets, ticket), proof, msgs, uint64(uts))
return m.api.MinerCreateBlock(context.TODO(), addr, base.ts, ticket, proof, msgs, nheight, uint64(uts))
}
type actorLookup func(context.Context, address.Address, *types.TipSet) (*types.Actor, error)

16
miner/testminer.go
View File

@ -2,14 +2,24 @@ package miner
import (
"context"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
)
func NewTestMiner(nextCh <-chan struct{}) func(api api) *Miner {
return func(api api) *Miner {
return &Miner{
func NewTestMiner(nextCh <-chan struct{}, addr address.Address) func(api.FullNode, gen.ElectionPoStProver) *Miner {
return func(api api.FullNode, epp gen.ElectionPoStProver) *Miner {
m := &Miner{
api: api,
waitFunc: chanWaiter(nextCh),
epp: epp,
}
if err := m.Register(addr); err != nil {
panic(err)
}
return m
}
}

7
node/builder.go
View File

@ -22,6 +22,7 @@ import (
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/blocksync"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/market"
"github.com/filecoin-project/lotus/chain/metrics"
"github.com/filecoin-project/lotus/chain/stmgr"
@ -92,7 +93,6 @@ const (
HandleDealsKey
HandleRetrievalKey
RunSectorServiceKey
RegisterMinerKey
RegisterProviderValidatorKey
// daemon
@ -231,8 +231,6 @@ func Online() Option {
Override(new(*paych.Store), paych.NewStore),
Override(new(*paych.Manager), paych.NewManager),
Override(new(*market.FundMgr), market.NewFundMgr),
Override(new(*miner.Miner), miner.NewMiner),
),
// Storage miner
@ -252,7 +250,8 @@ func Online() Option {
Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator),
Override(HandleRetrievalKey, modules.HandleRetrieval),
Override(HandleDealsKey, modules.HandleDeals),
Override(RegisterMinerKey, modules.RegisterMiner),
Override(new(gen.ElectionPoStProver), storage.NewElectionPoStProver),
Override(new(*miner.Miner), modules.SetupBlockProducer),
),
)
}

18
node/impl/full.go
View File

@ -1,8 +1,6 @@
package impl
import (
"context"
logging "github.com/ipfs/go-log"
"github.com/filecoin-project/lotus/node/impl/client"
@ -10,8 +8,6 @@ import (
"github.com/filecoin-project/lotus/node/impl/paych"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/impl/full"
)
@ -27,20 +23,6 @@ type FullNodeAPI struct {
full.StateAPI
full.WalletAPI
full.SyncAPI
Miner *miner.Miner
}
func (a *FullNodeAPI) MinerAddresses(context.Context) ([]address.Address, error) {
return a.Miner.Addresses()
}
func (a *FullNodeAPI) MinerRegister(ctx context.Context, addr address.Address) error {
return a.Miner.Register(addr)
}
func (a *FullNodeAPI) MinerUnregister(ctx context.Context, addr address.Address) error {
return a.Miner.Unregister(ctx, addr)
}
var _ api.FullNode = &FullNodeAPI{}

4
node/impl/full/chain.go
View File

@ -28,8 +28,8 @@ func (a *ChainAPI) ChainHead(context.Context) (*types.TipSet, error) {
return a.Chain.GetHeaviestTipSet(), nil
}
func (a *ChainAPI) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, tickets []*types.Ticket, lb int) ([]byte, error) {
return a.Chain.GetRandomness(ctx, pts.Cids(), tickets, int64(lb))
func (a *ChainAPI) ChainGetRandomness(ctx context.Context, pts types.TipSetKey, round int64) ([]byte, error) {
return a.Chain.GetRandomness(ctx, pts.Cids(), round)
}
func (a *ChainAPI) ChainGetBlock(ctx context.Context, msg cid.Cid) (*types.BlockHeader, error) {

8
node/impl/full/state.go
View File

@ -76,8 +76,8 @@ func (a *StateAPI) StateMinerPeerID(ctx context.Context, m address.Address, ts *
return stmgr.GetMinerPeerID(ctx, a.StateManager, ts, m)
}
func (a *StateAPI) StateMinerProvingPeriodEnd(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return stmgr.GetMinerProvingPeriodEnd(ctx, a.StateManager, ts, actor)
func (a *StateAPI) StateMinerElectionPeriodStart(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
return stmgr.GetMinerElectionPeriodStart(ctx, a.StateManager, ts, actor)
}
func (a *StateAPI) StateMinerSectorSize(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) {
@ -186,8 +186,8 @@ func (a *StateAPI) StateReadState(ctx context.Context, act *types.Actor, ts *typ
}
// This is on StateAPI because miner.Miner requires this, and MinerAPI requires miner.Miner
func (a *StateAPI) MinerCreateBlock(ctx context.Context, addr address.Address, parents *types.TipSet, tickets []*types.Ticket, proof types.ElectionProof, msgs []*types.SignedMessage, ts uint64) (*types.BlockMsg, error) {
fblk, err := gen.MinerCreateBlock(ctx, a.StateManager, a.Wallet, addr, parents, tickets, proof, msgs, ts)
func (a *StateAPI) MinerCreateBlock(ctx context.Context, addr address.Address, parents *types.TipSet, ticket *types.Ticket, proof *types.EPostProof, msgs []*types.SignedMessage, height, ts uint64) (*types.BlockMsg, error) {
fblk, err := gen.MinerCreateBlock(ctx, a.StateManager, a.Wallet, addr, parents, ticket, proof, msgs, height, ts)
if err != nil {
return nil, err
}

6
node/impl/storminer.go
View File

@ -6,6 +6,7 @@ import (
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/storage"
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
@ -17,8 +18,9 @@ type StorageMinerAPI struct {
SectorBuilder *sectorbuilder.SectorBuilder
SectorBlocks *sectorblocks.SectorBlocks
Miner *storage.Miner
Full api.FullNode
Miner *storage.Miner
BlockMiner *miner.Miner
Full api.FullNode
}
func (sm *StorageMinerAPI) WorkerStats(context.Context) (api.WorkerStats, error) {

53
node/modules/storageminer.go
View File

@ -2,7 +2,6 @@ package modules
import (
"context"
"fmt"
"math"
"path/filepath"
"reflect"
@ -24,9 +23,11 @@ import (
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/deals"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
@ -65,7 +66,7 @@ func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataD
}
cache := filepath.Join(sp, "cache")
metadata := filepath.Join(sp, "meta")
unsealed := filepath.Join(sp, "unsealed")
sealed := filepath.Join(sp, "sealed")
staging := filepath.Join(sp, "staging")
@ -75,7 +76,7 @@ func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataD
WorkerThreads: uint8(threads),
CacheDir: cache,
MetadataDir: metadata,
UnsealedDir: unsealed,
SealedDir: sealed,
StagedDir: staging,
}
@ -176,41 +177,35 @@ func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt
return dag, nil
}
func RegisterMiner(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode) error {
func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, epp gen.ElectionPoStProver) (*miner.Miner, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return err
}
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
log.Infof("Registering miner '%s' with full node", minerAddr)
if err := api.MinerRegister(ctx, minerAddr); err != nil {
return fmt.Errorf("Failed to register miner: %s\nIf you are certain no other storage miner instance is running, try running 'lotus unregister-miner %s' and restarting the storage miner", err, minerAddr)
}
return nil
},
OnStop: func(ctx context.Context) error {
log.Infof("Unregistering miner '%s' from full node", minerAddr)
return api.MinerUnregister(ctx, minerAddr)
},
})
return nil
}
func SectorBuilder(lc fx.Lifecycle, cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) {
sb, err := sectorbuilder.New(cfg, ds)
if err != nil {
return nil, err
}
m := miner.NewMiner(api, epp)
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
sb.Destroy()
OnStart: func(ctx context.Context) error {
if err := m.Register(minerAddr); err != nil {
return err
}
return nil
},
OnStop: func(ctx context.Context) error {
return m.Unregister(ctx, minerAddr)
},
})
return m, nil
}
func SectorBuilder(cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) {
sb, err := sectorbuilder.New(cfg, ds)
if err != nil {
return nil, err
}
return sb, nil
}
@ -221,7 +216,7 @@ func SealTicketGen(api api.FullNode) storage.TicketFn {
return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err)
}
r, err := api.ChainGetRandomness(ctx, ts.Key(), nil, build.SealRandomnessLookback)
r, err := api.ChainGetRandomness(ctx, ts.Key(), int64(ts.Height())-build.SealRandomnessLookback)
if err != nil {
return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err)
}

73
node/modules/testing/genesis.go
View File

@ -2,8 +2,11 @@ package testing
import (
"context"
"encoding/json"
"fmt"
"github.com/mitchellh/go-homedir"
"io"
"io/ioutil"
"os"
"time"
@ -13,35 +16,39 @@ import (
offline "github.com/ipfs/go-ipfs-exchange-offline"
logging "github.com/ipfs/go-log"
"github.com/ipfs/go-merkledag"
peer "github.com/libp2p/go-libp2p-peer"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/node/modules"
"github.com/filecoin-project/lotus/node/modules/dtypes"
)
var glog = logging.Logger("genesis")
func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
// TODO: make an address allocation
w, err := w.GenerateKey(types.KTBLS)
defk, err := w.GenerateKey(types.KTBLS)
if err != nil {
return nil, err
}
gmc := &gen.GenMinerCfg{
Owners: []address.Address{w},
Workers: []address.Address{w},
PeerIDs: []peer.ID{minerPid},
}
alloc := map[address.Address]types.BigInt{
w: types.FromFil(10000),
defk: types.FromFil(1000),
}
for _, genm := range gmc.PreSeals {
waddr, err := w.Import(&genm.Key)
if err != nil {
return nil, err
}
alloc[waddr] = types.FromFil(10000)
}
b, err := gen.MakeGenesisBlock(bs, alloc, gmc, 100000)
@ -61,23 +68,53 @@ func MakeGenesisMem(out io.Writer, minerPid peer.ID) func(bs dtypes.ChainBlockst
}
}
func MakeGenesis(outFile string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
func MakeGenesis(outFile, presealInfo string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
return func() (*types.BlockHeader, error) {
glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
minerAddr, err := w.GenerateKey(types.KTBLS)
presealInfo, err := homedir.Expand(presealInfo)
if err != nil {
return nil, err
}
gmc := &gen.GenMinerCfg{
Owners: []address.Address{minerAddr},
Workers: []address.Address{minerAddr},
PeerIDs: []peer.ID{"peer ID 1"},
fdata, err := ioutil.ReadFile(presealInfo)
if err != nil {
return nil, err
}
addrs := map[address.Address]types.BigInt{
minerAddr: types.FromFil(100000),
var preseal map[string]genesis.GenesisMiner
if err := json.Unmarshal(fdata, &preseal); err != nil {
return nil, err
}
minerAddresses := make([]address.Address, 0, len(preseal))
for s := range preseal {
a, err := address.NewFromString(s)
if err != nil {
return nil, err
}
if a.Protocol() != address.ID {
return nil, xerrors.New("expected ID address")
}
minerAddresses = append(minerAddresses, a)
}
gmc := &gen.GenMinerCfg{
PeerIDs: []peer.ID{"peer ID 1"},
PreSeals: preseal,
MinerAddrs: minerAddresses,
}
addrs := map[address.Address]types.BigInt{}
for _, miner := range preseal {
if _, err := w.Import(&miner.Key); err != nil {
return nil, xerrors.Errorf("importing miner key: %w", err)
}
_ = w.SetDefault(miner.Worker)
addrs[miner.Worker] = types.FromFil(100000)
}
b, err := gen.MakeGenesisBlock(bs, addrs, gmc, uint64(time.Now().Unix()))

110
node/node_test.go
View File

@ -4,12 +4,16 @@ import (
"bytes"
"context"
"crypto/rand"
"io/ioutil"
"net/http/httptest"
"path/filepath"
"testing"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
logging "github.com/ipfs/go-log"
"github.com/libp2p/go-libp2p-core/peer"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
"github.com/stretchr/testify/require"
@ -19,15 +23,24 @@ import (
"github.com/filecoin-project/lotus/api/test"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
"github.com/filecoin-project/lotus/genesis"
"github.com/filecoin-project/lotus/lib/jsonrpc"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/modules"
modtest "github.com/filecoin-project/lotus/node/modules/testing"
"github.com/filecoin-project/lotus/node/repo"
)
func init() {
_ = logging.SetLogLevel("*", "INFO")
}
func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, act address.Address, pk crypto.PrivKey, tnd test.TestNode, mn mocknet.Mocknet) test.TestStorageNode {
r := repo.NewMemory(nil)
@ -76,6 +89,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
// start node
var minerapi api.StorageMiner
mineBlock := make(chan struct{})
// TODO: use stop
_, err = node.New(ctx,
node.StorageMiner(&minerapi),
@ -86,8 +100,11 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
node.MockHost(mn),
node.Override(new(api.FullNode), tnd),
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
)
require.NoError(t, err)
if err != nil {
t.Fatalf("failed to construct node: %v", err)
}
/*// Bootstrap with full node
remoteAddrs, err := tnd.NetAddrsListen(ctx)
@ -95,8 +112,16 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a
err = minerapi.NetConnect(ctx, remoteAddrs)
require.NoError(t, err)*/
mineOne := func(ctx context.Context) error {
select {
case mineBlock <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
return test.TestStorageNode{minerapi}
return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne}
}
func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.TestStorageNode) {
@ -114,16 +139,46 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
var genbuf bytes.Buffer
if len(storage) > 1 {
panic("need more peer IDs")
}
// PRESEAL SECTION, TRY TO REPLACE WITH BETTER IN THE FUTURE
// TODO: would be great if there was a better way to fake the preseals
gmc := &gen.GenMinerCfg{
PeerIDs: []peer.ID{minerPid}, // TODO: if we have more miners, need more peer IDs
PreSeals: map[string]genesis.GenesisMiner{},
}
var presealDirs []string
for i := 0; i < len(storage); i++ {
maddr, err := address.NewIDAddress(300 + uint64(i))
if err != nil {
t.Fatal(err)
}
tdir, err := ioutil.TempDir("", "preseal-memgen")
if err != nil {
t.Fatal(err)
}
genm, err := seed.PreSeal(maddr, 1024, 1, tdir, []byte("make genesis mem random"))
if err != nil {
t.Fatal(err)
}
presealDirs = append(presealDirs, tdir)
gmc.MinerAddrs = append(gmc.MinerAddrs, maddr)
gmc.PreSeals[maddr.String()] = *genm
}
// END PRESEAL SECTION
for i := 0; i < nFull; i++ {
var genesis node.Option
if i == 0 {
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, minerPid))
genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, gmc))
} else {
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
mineBlock := make(chan struct{})
var err error
// TODO: Don't ignore stop
_, err = node.New(ctx,
@ -133,22 +188,12 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
node.MockHost(mn),
node.Test(),
node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock)),
genesis,
)
if err != nil {
t.Fatal(err)
}
fulls[i].MineOne = func(ctx context.Context) error {
select {
case mineBlock <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
}
for i, full := range storage {
@ -162,13 +207,36 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
f := fulls[full]
wa, err := f.WalletDefaultAddress(ctx)
require.NoError(t, err)
genMiner, err := address.NewFromString("t0101")
require.NoError(t, err)
genMiner := gmc.MinerAddrs[i]
wa := gmc.PreSeals[genMiner.String()].Worker
storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn)
sma := storers[i].StorageMiner.(*impl.StorageMinerAPI)
psd := presealDirs[i]
mds, err := badger.NewDatastore(filepath.Join(psd, "badger"), nil)
if err != nil {
t.Fatal(err)
}
osb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: 1024,
WorkerThreads: 2,
Miner: genMiner,
CacheDir: filepath.Join(psd, "cache"),
StagedDir: filepath.Join(psd, "staging"),
SealedDir: filepath.Join(psd, "sealed"),
UnsealedDir: filepath.Join(psd, "unsealed"),
}, mds)
if err != nil {
t.Fatal(err)
}
if err := sma.SectorBuilder.ImportFrom(osb); err != nil {
t.Fatal(err)
}
}
if err := mn.LinkAll(); err != nil {
@ -197,7 +265,6 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
if err != nil {
t.Fatal(err)
}
fulls[i].MineOne = a.MineOne
}
for i, a := range storaApis {
@ -210,6 +277,7 @@ func rpcBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test
if err != nil {
t.Fatal(err)
}
storers[i].MineOne = a.MineOne
}
return fulls, storers

2
node/repo/interface.go
View File

@ -47,6 +47,6 @@ type LockedRepo interface {
// KeyStore returns store of private keys for Filecoin transactions
KeyStore() (types.KeyStore, error)
// Path returns absolute path of the repo (or empty string if in-memory)
// Path returns absolute path of the repo
Path() string
}

11
paych/simple.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/address"
@ -37,7 +38,7 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am
smsg, err := pm.mpool.MpoolPushMessage(ctx, msg)
if err != nil {
return address.Undef, cid.Undef, err
return address.Undef, cid.Undef, xerrors.Errorf("initializing paych actor: %w", err)
}
mcid := smsg.Cid()
@ -46,7 +47,7 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am
// (tricky because we need to setup channel tracking before we know it's address)
mwait, err := pm.state.StateWaitMsg(ctx, mcid)
if err != nil {
return address.Undef, cid.Undef, err
return address.Undef, cid.Undef, xerrors.Errorf("wait msg: %w", err)
}
if mwait.Receipt.ExitCode != 0 {
@ -60,11 +61,11 @@ func (pm *Manager) createPaych(ctx context.Context, from, to address.Address, am
ci, err := pm.loadOutboundChannelInfo(ctx, paychaddr)
if err != nil {
return address.Undef, cid.Undef, err
return address.Undef, cid.Undef, xerrors.Errorf("loading channel info: %w", err)
}
if err := pm.store.trackChannel(ci); err != nil {
return address.Undef, cid.Undef, err
return address.Undef, cid.Undef, xerrors.Errorf("tracking channel: %w", err)
}
return paychaddr, mcid, nil
@ -108,7 +109,7 @@ func (pm *Manager) GetPaych(ctx context.Context, from, to address.Address, ensur
return ci.Control == from && ci.Target == to
})
if err != nil {
return address.Undef, cid.Undef, err
return address.Undef, cid.Undef, xerrors.Errorf("findChan: %w", err)
}
if ch != address.Undef {
// TODO: Track available funds

2
retrieval/miner.go
View File

@ -41,7 +41,7 @@ func NewMiner(sblks *sectorblocks.SectorBlocks, full api.FullNode) *Miner {
}
func writeErr(stream network.Stream, err error) {
log.Errorf("Retrieval deal error: %s", err)
log.Errorf("Retrieval deal error: %+v", err)
_ = cborutil.WriteCborRPC(stream, &DealResponse{
Status: Error,
Message: err.Error(),

Some files were not shown because too many files have changed in this diff Show More