Merge pull request #225 from filecoin-project/feat/update-sectorbuilder

Update go-sectorbuilder
This commit is contained in:
Łukasz Magiera 2019-09-23 19:47:58 +02:00 committed by GitHub
commit 5f88fcb4f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 214 additions and 112 deletions

View File

@ -2,6 +2,10 @@ version: 2.1
orbs:
go: gotest/tools@0.0.9
executors:
golang:
docker:
- image: circleci/golang:1.13
commands:
install-deps:
@ -22,9 +26,6 @@ commands:
- 'v2-lotus-params-'
paths:
- /var/tmp/filecoin-proof-parameters/
- run:
command: GO111MODULE=on go get github.com/ipfs/ipget@c0cbd7d9d1925965a5aa0895d5ff32dbdb4a009e
working_directory: "~"
- run: make build/.params-1024
- save_cache:
name: Save parameters cache
@ -35,7 +36,7 @@ commands:
jobs:
mod-tidy-check:
executor: go/circleci-golang
executor: golang
steps:
- install-deps
- prepare
@ -48,7 +49,7 @@ jobs:
parameters:
executor:
type: executor
default: go/circleci-golang
default: golang
go-test-flags:
type: string
default: ""
@ -116,7 +117,7 @@ jobs:
parameters:
executor:
type: executor
default: go/circleci-golang
default: golang
golangci-lint-version:
type: string
default: 1.17.1

View File

@ -51,9 +51,9 @@ build/.update-modules:
## PROOFS
bin/ipget:
./bin/dist_get /ipns/dist.ipfs.io ipget bin/ipget v0.4.0
./bin/dist_get /ipns/dist.ipfs.io ipget bin/ipget v0.5.0
PARAM_SECTOR_SIZES:=1024 268435456
PARAM_SECTOR_SIZES:=1024 16777216 268435456 1073741824
PARAM_SECTOR_SIZES:=$(addprefix build/.params-,$(PARAM_SECTOR_SIZES))
$(PARAM_SECTOR_SIZES): build/proof-params/parameters.json bin/ipget

View File

@ -193,6 +193,8 @@ type MsgWait struct {
type BlockMessages struct {
BlsMessages []*types.Message
SecpkMessages []*types.SignedMessage
Cids []cid.Cid
}
type SectorInfo struct {

View File

@ -1,4 +1,24 @@
{
"v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.params": {
"cid": "QmZ6Y88jRbRjjYQzhh8o85bcUeChj7NGyo9yK6VbywhJ9F",
"digest": "3d245479d9b5fc668d58c493da5f3ee1",
"sector_size": 16777216
},
"v12-proof-of-spacetime-rational-535d1050e3adca2a0dfe6c3c0c4fa12097c9a7835fb969042f82a507b13310e0.vk": {
"cid": "QmbGgBLMTCnRc1E1fsUCPyZE4SYzrtAYEWCqCtL55AgxE2",
"digest": "29bd4c152096f878f41257b433159d81",
"sector_size": 16777216
},
"v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.params": {
"cid": "Qmdm8vhWeRsZUUaHdysDr91gv6u6RFeC18hHxGnnzPrwcW",
"digest": "c67fd415a65e6d1caf4278597cf3462e",
"sector_size": 1073741824
},
"v12-proof-of-spacetime-rational-b99f15d0bdaaf4ffb68b2ca72b69ea8d915f66a2a56f667430ad69d87aa5febd.vk": {
"cid": "Qmc3xssw1syaZDZKmiM7TbCRnoDPfmD6V8ec1eTESidJQS",
"digest": "870355c10000010b9a4ece80892acca2",
"sector_size": 1073741824
},
"v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
"cid": "QmZBvF2F9wTYKLBxWSCQKe34D3M7vkNNc7ou8mxnNhZkZc",
"digest": "5d854e0ecfbd12cb7fa1247a6e6a0315",
@ -19,6 +39,16 @@
"digest": "c83eca165ba94233861227578d658a22",
"sector_size": 268435456
},
"v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.params": {
"cid": "QmTkeyz3mfec4MqCbbiwuVWAQicY231zpdxSjdxN5U84PD",
"digest": "cf7118ac2273e2ccb6b451a5acd5f6e0",
"sector_size": 1073741824
},
"v12-zigzag-proof-of-replication-0ed875801b4a99e4a6b46e58703e6857277ce020510fc9080041f6cfd5e0d286.vk": {
"cid": "QmaGzJCwJNpQAD63QAKb8y4MKS1AMHeKyYzg3C5WyRSRRM",
"digest": "288c792f4fe09c85f8f35673fb9d5ed0",
"sector_size": 1073741824
},
"v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.params": {
"cid": "QmNSuxq15JPFCTehxVpgJydNZ79rpLoNwnLzQMGA9EziXg",
"digest": "818cd9cc2e0e47210a05bd073847ab5a",
@ -29,6 +59,16 @@
"digest": "a6636e2ee1a176161e022296bc045e79",
"sector_size": 268435456
},
"v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.params": {
"cid": "QmUBpwbVwu5uzjTS8n1yesJ2QUaVbWJ8D9p2VaSsfjgAUr",
"digest": "e7aa73a1b06290d30f567bfac8324bf1",
"sector_size": 16777216
},
"v12-zigzag-proof-of-replication-6496c180c2eab89ee0638bc73879ced01daf512486eb38b9e1c9402ba578e010.vk": {
"cid": "Qme2uK8sQJUT3DzF291p2b91eFQzn7cKSJFiKVBsvrjTgt",
"digest": "d318cd116803c8ccbd3ac4cded9400ad",
"sector_size": 16777216
},
"v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.params": {
"cid": "QmTfhTnkFvbpFfw8UydFdnPCDfxgAxEcw4fRdGsELpcFnh",
"digest": "906b6c0c9dc5bb581d9641c11b54e197",

View File

@ -247,7 +247,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
futurePower := types.BigAdd(self.Power, mi.SectorSize)
collateralRequired := CollateralForPower(futurePower)
if types.BigCmp(collateralRequired, act.Balance) < 0 {
if act.Balance.LessThan(collateralRequired) {
return nil, aerrors.New(3, "not enough collateral")
}
@ -328,11 +328,11 @@ func (sma StorageMinerActor) SubmitPoSt(act *types.Actor, vmctx types.VMContext,
//TODO temporary sector failure fees
msgVal := vmctx.Message().Value
if types.BigCmp(msgVal, feesRequired) < 0 {
if msgVal.LessThan(feesRequired) {
return nil, aerrors.New(2, "not enough funds to pay post submission fees")
}
if types.BigCmp(msgVal, feesRequired) > 0 {
if msgVal.GreaterThan(feesRequired) {
_, err := vmctx.Send(vmctx.Message().From, 0,
types.BigSub(msgVal, feesRequired), nil)
if err != nil {
@ -719,7 +719,7 @@ func (sma StorageMinerActor) SlashConsensusFault(act *types.Actor, vmctx types.V
}
slashedCollateral := params.SlashedCollateral
if types.BigCmp(slashedCollateral, act.Balance) < 0 {
if slashedCollateral.LessThan(act.Balance) {
slashedCollateral = act.Balance
}

View File

@ -173,12 +173,12 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types.
ls.Redeemed = sv.Amount
newSendBalance := types.BigAdd(self.ToSend, balanceDelta)
if types.BigCmp(newSendBalance, types.NewInt(0)) < 0 {
if newSendBalance.LessThan(types.NewInt(0)) {
// TODO: is this impossible?
return nil, aerrors.New(9, "voucher would leave channel balance negative")
}
if types.BigCmp(newSendBalance, act.Balance) > 0 {
if newSendBalance.GreaterThan(act.Balance) {
return nil, aerrors.New(10, "not enough funds in channel to cover voucher")
}

View File

@ -74,7 +74,7 @@ func (sma StorageMarketActor) CreateStorageMiner(act *types.Actor, vmctx types.V
return nil, err
}
if types.BigCmp(vmctx.Message().Value, reqColl) < 0 {
if vmctx.Message().Value.LessThan(reqColl) {
return nil, aerrors.Newf(1, "not enough funds passed to cover required miner collateral (needed %s, got %s)", reqColl, vmctx.Message().Value)
}

View File

@ -2,9 +2,9 @@ package deals
import (
"context"
"github.com/filecoin-project/go-lotus/lib/sectorbuilder"
"runtime"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
"github.com/ipfs/go-cid"
files "github.com/ipfs/go-ipfs-files"
cbor "github.com/ipfs/go-ipld-cbor"
@ -56,11 +56,11 @@ func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, int64, error)
return nil, 0, err
}
var commP [sectorbuilder.CommitmentBytesLen]byte
err = withTemp(uf, func(f string) error {
commP, err = sectorbuilder.GeneratePieceCommitment(f, uint64(size))
return err
})
commP, err := sectorbuilder.GeneratePieceCommitment(uf, uint64(size))
if err != nil {
return nil, 0, err
}
return commP[:], size, err
}

View File

@ -5,6 +5,7 @@ import (
"context"
"github.com/filecoin-project/go-lotus/api"
"github.com/filecoin-project/go-lotus/build"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/ipfs/go-merkledag"
@ -90,7 +91,7 @@ func (h *Handler) validateVouchers(ctx context.Context, deal MinerDeal) error {
// TODO: make sure that current laneStatus.Amount == 0
if types.BigCmp(voucher.Amount, deal.Proposal.TotalPrice) < 0 {
if voucher.Amount.LessThan(deal.Proposal.TotalPrice) {
return xerrors.Errorf("validating payment voucher %d: not enough funds in the voucher", i)
}
@ -202,13 +203,13 @@ func (h *Handler) waitSealed(ctx context.Context, deal MinerDeal) (sectorbuilder
return sectorbuilder.SectorSealingStatus{}, err
}
switch status.SealStatusCode {
case 0: // sealed
case 2: // failed
switch status.State {
case sealing_state.Sealed:
case sealing_state.Failed:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sealing sector %d for deal %s (ref=%s) failed: %s", deal.SectorID, deal.ProposalCid, deal.Ref, status.SealErrorMsg)
case 1: // pending
case sealing_state.Pending:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'pending' after call to WaitSeal (for sector %d)", deal.SectorID)
case 3: // sealing
case sealing_state.Sealing:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("sector status was 'wait' after call to WaitSeal (for sector %d)", deal.SectorID)
default:
return sectorbuilder.SectorSealingStatus{}, xerrors.Errorf("unknown SealStatusCode: %d", status.SectorID)

View File

@ -1,28 +0,0 @@
package deals
import (
"io"
"io/ioutil"
"os"
)
func withTemp(r io.Reader, cb func(string) error) error {
f, err := ioutil.TempFile(os.TempDir(), "lotus-temp-")
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
err = cb(f.Name())
if err != nil {
os.Remove(f.Name())
return err
}
return os.Remove(f.Name())
}

View File

@ -85,6 +85,16 @@ func (bi *BigInt) Nil() bool {
return bi.Int == nil
}
// LessThan returns true if bi < o
func (bi *BigInt) LessThan(o BigInt) bool {
return BigCmp(*bi, o) < 0
}
// LessThan returns true if bi > o
func (bi *BigInt) GreaterThan(o BigInt) bool {
return BigCmp(*bi, o) > 0
}
func (bi *BigInt) MarshalJSON() ([]byte, error) {
return json.Marshal(bi.String())
}

View File

@ -171,5 +171,6 @@ func PowerCmp(eproof ElectionProof, mpow, totpow BigInt) bool {
top := BigMul(rden, mpow)
out := BigDiv(top, totpow)
return BigCmp(BigFromBytes(h[:]), out) < 0
hp := BigFromBytes(h[:])
return hp.LessThan(out)
}

View File

@ -167,7 +167,7 @@ func (vmc *VMContext) GasUsed() types.BigInt {
func (vmc *VMContext) ChargeGas(amount uint64) aerrors.ActorError {
toUse := types.NewInt(amount)
vmc.gasUsed = types.BigAdd(vmc.gasUsed, toUse)
if types.BigCmp(vmc.gasUsed, vmc.gasAvailable) > 0 {
if vmc.gasUsed.GreaterThan(vmc.gasAvailable) {
return aerrors.Newf(outOfGasErrCode, "not enough gas: used=%s, available=%s", vmc.gasUsed, vmc.gasAvailable)
}
return nil
@ -451,7 +451,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet,
gascost := types.BigMul(msg.GasLimit, msg.GasPrice)
totalCost := types.BigAdd(gascost, msg.Value)
if types.BigCmp(fromActor.Balance, totalCost) < 0 {
if fromActor.Balance.LessThan(totalCost) {
return nil, xerrors.Errorf("not enough funds (%s < %s)", fromActor.Balance, totalCost)
}
if err := DeductFunds(fromActor, gascost); err != nil {
@ -619,7 +619,7 @@ func (vm *VM) Invoke(act *types.Actor, vmctx *VMContext, method uint64, params [
}
func DeductFunds(act *types.Actor, amt types.BigInt) error {
if types.BigCmp(act.Balance, amt) < 0 {
if act.Balance.LessThan(amt) {
return fmt.Errorf("not enough funds")
}

View File

@ -291,7 +291,7 @@ var paychVoucherBestSpendableCmd = &cli.Command{
return err
}
if spendable {
if best == nil || types.BigCmp(v.Amount, best.Amount) > 0 {
if best == nil || v.Amount.GreaterThan(best.Amount) {
best = v
}
}

View File

@ -65,7 +65,7 @@ var sectorsStatusCmd = &cli.Command{
}
fmt.Printf("SectorID:\t%d\n", status.SectorID)
fmt.Printf("SealStatusCode:\t%d\n", status.SealStatusCode)
fmt.Printf("Status:\t%s\n", status.State.String())
fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg)
fmt.Printf("CommD:\t\t%x\n", status.CommD)
fmt.Printf("CommR:\t\t%x\n", status.CommR)

@ -1 +1 @@
Subproject commit 632a680cdb85ac50e6f6bbae25eda46f276c4a7b
Subproject commit 641e9ea587a583a6dcdc74694109a6f7e21e2980

2
go.mod
View File

@ -1,6 +1,6 @@
module github.com/filecoin-project/go-lotus
go 1.12
go 1.13
require (
contrib.go.opencensus.io/exporter/jaeger v0.1.0

View File

@ -2,6 +2,9 @@ package sectorbuilder
import (
"encoding/binary"
"io"
"os"
"sync"
"unsafe"
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
@ -64,8 +67,18 @@ func (sb *SectorBuilder) Destroy() {
sectorbuilder.DestroySectorBuilder(sb.handle)
}
func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, piecePath string) (uint64, error) {
return sectorbuilder.AddPiece(sb.handle, pieceKey, pieceSize, piecePath)
func (sb *SectorBuilder) AddPiece(pieceKey string, pieceSize uint64, file io.Reader) (uint64, error) {
f, werr, err := toReadableFile(file, int64(pieceSize))
if err != nil {
return 0, err
}
sectorID, err := sectorbuilder.AddPieceFromFile(sb.handle, pieceKey, pieceSize, f)
if err != nil {
return 0, err
}
return sectorID, werr()
}
// TODO: should *really really* return an io.ReadCloser
@ -118,3 +131,51 @@ func NewSortedSectorInfo(sectors []SectorInfo) SortedSectorInfo {
func VerifyPost(sectorSize uint64, sectorInfo SortedSectorInfo, challengeSeed [CommLen]byte, proof []byte, faults []uint64) (bool, error) {
return sectorbuilder.VerifyPoSt(sectorSize, sectorInfo, challengeSeed, proof, faults)
}
func GeneratePieceCommitment(piece io.Reader, pieceSize uint64) (commP [CommLen]byte, err error) {
f, werr, err := toReadableFile(piece, int64(pieceSize))
if err != nil {
return [32]byte{}, err
}
commP, err = sectorbuilder.GeneratePieceCommitmentFromFile(f, pieceSize)
if err != nil {
return [32]byte{}, err
}
return commP, werr()
}
func toReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
f, ok := r.(*os.File)
if ok {
return f, func() error { return nil }, nil
}
var w *os.File
f, w, err := os.Pipe()
if err != nil {
return nil, nil, err
}
var wait sync.Mutex
var werr error
wait.Lock()
go func() {
defer wait.Unlock()
_, werr = io.CopyN(w, r, n)
err := w.Close()
if werr == nil {
werr = err
}
}()
return f, func() error {
wait.Lock()
return werr
}, nil
}

View File

@ -34,15 +34,9 @@ func TestSealAndVerify(t *testing.T) {
t.Fatal(err)
}
fi, err := ioutil.TempFile("", "sbtestfi")
if err != nil {
t.Fatal(err)
}
defer fi.Close()
r := io.LimitReader(rand.New(rand.NewSource(42)), 1016)
io.CopyN(fi, rand.New(rand.NewSource(42)), 1016)
if _, err := sb.AddPiece("foo", 1016, fi.Name()); err != nil {
if _, err := sb.AddPiece("foo", 1016, r); err != nil {
t.Fatal(err)
}

View File

@ -117,6 +117,23 @@ a:hover {
/* POND */
.Pond-connecting {
width: 100vw;
height: 100vh;
background: #1a1a1a;
color: #ffffff;
font-family: monospace;
display: grid;
grid-template-columns: auto min-content auto;
grid-template-rows: auto min-content auto;
grid-template-areas:
". . ."
". main ."
". . ."
}
.App {
min-height: 100vh;
background: #1a1a1a;

View File

@ -17,6 +17,8 @@ class Block extends React.Component {
let messages = await this.props.conn.call('Filecoin.ChainGetBlockMessages', [this.props.cid])
let receipts = await this.props.conn.call('Filecoin.ChainGetBlockReceipts', [this.props.cid])
const mcids = messages.Cids
messages = [
...(messages.BlsMessages.map(m => ({...m, type: 'BLS'}))),
...(messages.SecpkMessages.map(m => ({...(m.Message), type: 'Secpk'})))
@ -24,6 +26,17 @@ class Block extends React.Component {
messages = messages.map((msg, k) => ({...msg, receipt: receipts[k]}))
messages = await Promise.all(messages.map(async (msg, i) => {
if (msg.receipt.ExitCode !== 0) {
let reply = await this.props.conn.call('Filecoin.StateReplay', [{Cids: [this.props.cid], Blocks: [header], Height: header.Height}, mcids[i]])
if(!reply.Error) {
reply.Error = "reply: no error"
}
msg.Error = reply.Error
}
return msg
}))
this.setState({header: header, messages: messages})
}
@ -33,6 +46,7 @@ class Block extends React.Component {
let head = this.state.header
const messages = this.state.messages.map(m => (
<div>
<div>
<Address client={this.props.conn} addr={m.From} mountWindow={this.props.mountWindow}/><b>&nbsp;=>&nbsp;</b>
<Address client={this.props.conn} addr={m.To} mountWindow={this.props.mountWindow} transfer={m.Value} method={m.Method}/>
@ -40,6 +54,8 @@ class Block extends React.Component {
<span>&nbsp;{m.receipt.GasUsed}Gas</span>
{m.receipt.ExitCode !== 0 ? <span>&nbsp;<b>EXIT:{m.receipt.ExitCode}</b></span> : <span/>}
</div>
{m.receipt.ExitCode !== 0 ? <div>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Error: <b>{m.Error}</b></div> : <span/>}
</div>
))
content = (

View File

@ -35,8 +35,10 @@ class Pond extends React.Component {
render() {
if (this.state.client === undefined) {
return (
<div className="Pond-connecting">
<div>
Connecting to RPC
<div>Connecting to Pond RPC</div>
</div>
</div>
)
}

View File

@ -77,9 +77,20 @@ func (a *ChainAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api
return nil, err
}
cids := make([]cid.Cid, len(bmsgs)+len(smsgs))
for i, m := range bmsgs {
cids[i] = m.Cid()
}
for i, m := range smsgs {
cids[i+len(bmsgs)] = m.Cid()
}
return &api.BlockMessages{
BlsMessages: bmsgs,
SecpkMessages: smsgs,
Cids: cids,
}, nil
}

View File

@ -159,7 +159,7 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv
// TODO: also account for vouchers on other lanes we've received
newTotal := types.BigAdd(sendAmount, pca.ToSend)
if types.BigCmp(act.Balance, newTotal) < 0 {
if act.Balance.LessThan(newTotal) {
return fmt.Errorf("not enough funds in channel to cover voucher")
}

View File

@ -226,7 +226,7 @@ func (ps *Store) AddVoucher(ch address.Address, sv *types.SignedVoucher, proof [
}
delta := types.BigSub(sv.Amount, bestAmount)
if types.BigCmp(minDelta, delta) > 0 {
if minDelta.GreaterThan(delta) {
return delta, xerrors.Errorf("addVoucher: supplied token amount too low; minD=%s, D=%s; bestAmt=%s; v.Amt=%s", minDelta, delta, bestAmount, sv.Amount)
}

View File

@ -2,10 +2,9 @@ package sector
import (
"context"
"github.com/filecoin-project/go-sectorbuilder/sealing_state"
"golang.org/x/xerrors"
"io"
"io/ioutil"
"os"
"sync"
"time"
@ -63,7 +62,7 @@ func (s *Store) poll() {
continue
}
if status.SealStatusCode == 0 { // constant pls, zero implies the last step?
if status.State == sealing_state.Sealed {
done = append(done, status)
}
}
@ -102,10 +101,8 @@ func (s *Store) service() {
}
func (s *Store) AddPiece(ref string, size uint64, r io.Reader) (sectorID uint64, err error) {
err = withTemp(r, func(f string) (err error) {
sectorID, err = s.sb.AddPiece(ref, size, f)
return err
})
sectorID, err = s.sb.AddPiece(ref, size, r)
if err != nil {
return 0, err
}
@ -191,26 +188,3 @@ func (s *Store) RunPoSt(ctx context.Context, sectors []*api.SectorInfo, r []byte
func (s *Store) Stop() {
close(s.closeCh)
}
func withTemp(r io.Reader, cb func(string) error) error {
f, err := ioutil.TempFile(os.TempDir(), "lotus-temp-")
if err != nil {
return err
}
if _, err := io.Copy(f, r); err != nil {
return err
}
if err := f.Close(); err != nil {
return err
}
err = cb(f.Name())
if err != nil {
if err := os.Remove(f.Name()); err != nil {
log.Errorf("couldn't remove temp file '%s'", f.Name())
}
return err
}
return os.Remove(f.Name())
}