Merge pull request #163 from filecoin-project/feat/sector-set

switch to using AMT for sector set
This commit is contained in:
Whyrusleeping 2019-08-29 09:56:43 -07:00 committed by GitHub
commit 55abcaa7d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 79 additions and 60 deletions

View File

@ -7,8 +7,10 @@ import (
"github.com/filecoin-project/go-lotus/chain/address"
"github.com/filecoin-project/go-lotus/chain/types"
"github.com/filecoin-project/go-lotus/lib/sectorbuilder"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-hamt-ipld"
amt "github.com/filecoin-project/go-amt-ipld"
cid "github.com/ipfs/go-cid"
hamt "github.com/ipfs/go-hamt-ipld"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/libp2p/go-libp2p-core/peer"
"golang.org/x/xerrors"
@ -202,7 +204,7 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ
}
type CommitSectorParams struct {
SectorID types.BigInt
SectorID uint64
CommD []byte
CommR []byte
CommRStar []byte
@ -210,6 +212,7 @@ type CommitSectorParams struct {
}
func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *CommitSectorParams) ([]byte, ActorError) {
ctx := context.TODO()
oldstate, self, err := loadState(vmctx)
if err != nil {
return nil, err
@ -230,7 +233,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
}
// make sure the miner isnt trying to submit a pre-existing sector
unique, err := SectorIsUnique(vmctx.Ipld(), self.Sectors, params.SectorID)
unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorID)
if err != nil {
return nil, err
}
@ -254,7 +257,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex
// Note: There must exist a unique index in the miner's sector set for each
// sector ID. The `faults`, `recovered`, and `done` parameters of the
// SubmitPoSt method express indices into this sector set.
nssroot, err := AddToSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, params.SectorID, params.CommR, params.CommD)
nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, params.CommR, params.CommD)
if err != nil {
return nil, err
}
@ -344,74 +347,48 @@ func (sma StorageMinerActor) GetPower(act *types.Actor, vmctx types.VMContext, p
return self.Power.Bytes(), nil
}
func SectorIsUnique(cst *hamt.CborIpldStore, sroot cid.Cid, sid types.BigInt) (bool, ActorError) {
nd, err := hamt.LoadNode(context.TODO(), cst, sroot)
func SectorIsUnique(ctx context.Context, s types.Storage, sroot cid.Cid, sid uint64) (bool, ActorError) {
found, _, _, err := GetFromSectorSet(ctx, s, sroot, sid)
if err != nil {
return false, aerrors.Escalate(err, "could not load node in HAMT")
return false, err
}
if _, err := nd.Find(context.TODO(), sid.String()); err != nil {
if xerrors.Is(err, hamt.ErrNotFound) {
return true, nil
}
return false, aerrors.Escalate(err, "could not find node in HAMT")
}
return false, nil
return !found, nil
}
func AddToSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid, sectorID types.BigInt, commR, commD []byte) (cid.Cid, ActorError) {
nd, err := hamt.LoadNode(ctx, cst, ss)
func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) {
ssr, err := amt.LoadAMT(types.WrapStorage(s), ss)
if err != nil {
return cid.Undef, aerrors.Escalate(err, "could not load HAMT node")
return cid.Undef, aerrors.Escalate(err, "could not load sector set node")
}
enc, aerr := SerializeParams([][]byte{commR, commD})
if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil {
return cid.Undef, aerrors.Escalate(err, "failed to set commitment in sector set")
}
ncid, err := ssr.Flush()
if err != nil {
return cid.Undef, aerrors.Wrap(aerr, "failed to serialize commR and commD for sector set")
}
if err := nd.Set(ctx, sectorID.String(), enc); err != nil {
return cid.Undef, aerrors.Escalate(err, "failed to set new sector in sector set")
}
if err := nd.Flush(ctx); err != nil {
return cid.Undef, aerrors.Escalate(err, "failed to flush sector set")
}
ssroot, err := cst.Put(ctx, nd)
if err != nil {
return cid.Undef, aerrors.Escalate(err, "failed to store new sector set root")
}
return ssroot, nil
return ncid, nil
}
func GetFromSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid, sectorID types.BigInt) (bool, []byte, []byte, ActorError) {
nd, err := hamt.LoadNode(ctx, cst, ss)
func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64) (bool, []byte, []byte, ActorError) {
ssr, err := amt.LoadAMT(types.WrapStorage(s), ss)
if err != nil {
return false, nil, nil, aerrors.Escalate(err, "could not load HAMT node")
return false, nil, nil, aerrors.Escalate(err, "could not load sector set node")
}
infoIf, err := nd.Find(ctx, sectorID.String())
if err == hamt.ErrNotFound {
return false, nil, nil, nil
}
var comms [][]byte
err = ssr.Get(sectorID, &comms)
if err != nil {
if _, ok := err.(amt.ErrNotFound); ok {
return false, nil, nil, nil
}
return false, nil, nil, aerrors.Escalate(err, "failed to find sector in sector set")
}
infoB, ok := infoIf.([]byte)
if !ok {
return false, nil, nil, aerrors.Escalate(xerrors.New("casting infoIf to []byte failed"), "") // TODO: Review: how to create aerrror without retcode?
}
var comms [][]byte // [ [commR], [commD] ]
err = cbor.DecodeInto(infoB, &comms)
if err != nil {
return false, nil, nil, aerrors.Escalate(err, "failed to decode sector set entry")
}
if len(comms) != 2 {
return false, nil, nil, aerrors.Escalate(xerrors.New("sector set entry should only have 2 elements"), "")
}
@ -420,7 +397,7 @@ func GetFromSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid,
}
func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *CommitSectorParams) (bool, ActorError) {
ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorID.Uint64(), params.Proof)
ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorID, params.Proof)
if err != nil {
return false, aerrors.Escalate(err, "verify seal failed")
}
@ -548,7 +525,7 @@ type PieceInclVoucherData struct { // TODO: Update spec at https://github.com/fi
}
type InclusionProof struct {
Sector types.BigInt // for CommD, also verifies the sector is in sector set
Sector uint64 // for CommD, also verifies the sector is in sector set
Proof []byte
}
@ -574,7 +551,7 @@ func (sma StorageMinerActor) PaymentVerifyInclusion(act *types.Actor, vmctx type
return nil, aerrors.Escalate(err, "failed to decode storage payment proof")
}
ok, _, commD, aerr := GetFromSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, proof.Sector)
ok, _, commD, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, proof.Sector)
if aerr != nil {
return nil, aerr
}
@ -609,7 +586,7 @@ func (sma StorageMinerActor) PaymentVerifySector(act *types.Actor, vmctx types.V
return nil, aerrors.New(1, "unexpected proof bytes")
}
ok, _, _, aerr := GetFromSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, sector)
ok, _, _, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, sector.Uint64())
if aerr != nil {
return nil, aerr
}

View File

@ -1,6 +1,7 @@
package types
import (
"github.com/filecoin-project/go-amt-ipld"
"github.com/filecoin-project/go-lotus/chain/actors/aerrors"
"github.com/filecoin-project/go-lotus/chain/address"
"github.com/ipfs/go-cid"
@ -35,3 +36,28 @@ type VMContext interface {
VerifySignature(sig *Signature, from address.Address, data []byte) aerrors.ActorError
ChargeGas(uint64) aerrors.ActorError
}
type storageWrapper struct {
s Storage
}
func (sw *storageWrapper) Put(i interface{}) (cid.Cid, error) {
c, err := sw.s.Put(i)
if err != nil {
return cid.Undef, err
}
return c, nil
}
func (sw *storageWrapper) Get(c cid.Cid, out interface{}) error {
if err := sw.s.Get(c, out); err != nil {
return err
}
return nil
}
func WrapStorage(s Storage) amt.Blocks {
return &storageWrapper{s}
}

7
go.mod
View File

@ -5,6 +5,7 @@ go 1.12
require (
contrib.go.opencensus.io/exporter/jaeger v0.1.0
github.com/BurntSushi/toml v0.3.1
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985
github.com/filecoin-project/go-bls-sigs v0.0.0-20190718224239-4bc4b8a7bbf8
github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543
github.com/filecoin-project/go-sectorbuilder v0.0.0-00010101000000-000000000000
@ -15,12 +16,12 @@ require (
github.com/ipfs/go-blockservice v0.1.2
github.com/ipfs/go-car v0.0.1
github.com/ipfs/go-cid v0.0.3
github.com/ipfs/go-datastore v0.0.5
github.com/ipfs/go-datastore v0.1.0
github.com/ipfs/go-ds-badger v0.0.5
github.com/ipfs/go-filestore v0.0.2
github.com/ipfs/go-fs-lock v0.0.1
github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf
github.com/ipfs/go-ipfs-blockstore v0.0.1
github.com/ipfs/go-ipfs-blockstore v0.1.0
github.com/ipfs/go-ipfs-chunker v0.0.1
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/ipfs/go-ipfs-exchange-offline v0.0.1
@ -66,7 +67,7 @@ require (
github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect
github.com/stretchr/objx v0.1.1 // indirect
github.com/stretchr/testify v1.3.0
github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804
github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33

14
go.sum
View File

@ -64,6 +64,12 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828222406-5f4c30401bda h1:IZKqXB3VvlRCwroy8GbdHjyxJgueveGPXKiSL8nvI7o=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828222406-5f4c30401bda/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828225523-8bbaf27f2782 h1:J0h4pFlx2ZSm1xWO7e2EesXz5xW/35AMqMzk4VGZ6B8=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828225523-8bbaf27f2782/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985 h1:rpid5Xgp6GnDACqZvugxWvJ8L1HpyttUkzpUk/4BPXk=
github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY=
github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug=
github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543/go.mod h1:mjrHv1cDGJWDlGmC0eDc1E5VJr8DmL9XMUcaFwiuKg8=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -120,6 +126,8 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw=
github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c=
github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs=
@ -142,6 +150,8 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo=
github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI=
github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
@ -157,6 +167,8 @@ github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf h1:P9Kkd8YCG4
github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf/go.mod h1:gaK14QN1GOlYGgq+o+t5+WTExZZogkMt0k0IIBNjXsM=
github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc=
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88=
github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw=
@ -508,6 +520,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1 h1:wDIXmhgP
github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804 h1:wK83hcnZgKf1AyH8804pc4zVZOB2ND+d2cagsuhQmZc=
github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a h1:9oEQR9eq2H2JDmglMcrCa+TxUEYy3HKSiNoLIkPNy/U=
github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=

View File

@ -3,6 +3,7 @@ package storage
import (
"context"
"fmt"
"github.com/filecoin-project/go-lotus/storage/sector"
"github.com/filecoin-project/go-lotus/api"
@ -111,7 +112,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal
}
params := &actors.CommitSectorParams{
SectorID: types.NewInt(sinfo.SectorID),
SectorID: sinfo.SectorID,
CommD: sinfo.CommD[:],
CommR: sinfo.CommR[:],
CommRStar: sinfo.CommRStar[:],