From 2013db604a941d54277b4236ea4bb5b8d276d152 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 28 Aug 2019 17:01:46 -0700 Subject: [PATCH] switch to using AMT for sector set --- chain/actors/actor_miner.go | 89 ++++++++++++++----------------------- chain/types/vmcontext.go | 26 +++++++++++ go.mod | 7 +-- go.sum | 14 ++++++ storage/miner.go | 3 +- 5 files changed, 79 insertions(+), 60 deletions(-) diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 1eac74ed1..46fbcdb99 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -7,8 +7,10 @@ import ( "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/types" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-hamt-ipld" + + amt "github.com/filecoin-project/go-amt-ipld" + cid "github.com/ipfs/go-cid" + hamt "github.com/ipfs/go-hamt-ipld" cbor "github.com/ipfs/go-ipld-cbor" "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" @@ -202,7 +204,7 @@ func (sma StorageMinerActor) StorageMinerConstructor(act *types.Actor, vmctx typ } type CommitSectorParams struct { - SectorID types.BigInt + SectorID uint64 CommD []byte CommR []byte CommRStar []byte @@ -210,6 +212,7 @@ type CommitSectorParams struct { } func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContext, params *CommitSectorParams) ([]byte, ActorError) { + ctx := context.TODO() oldstate, self, err := loadState(vmctx) if err != nil { return nil, err @@ -230,7 +233,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex } // make sure the miner isnt trying to submit a pre-existing sector - unique, err := SectorIsUnique(vmctx.Ipld(), self.Sectors, params.SectorID) + unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorID) if err != nil { return nil, err } @@ -254,7 +257,7 @@ func (sma StorageMinerActor) CommitSector(act *types.Actor, vmctx types.VMContex // Note: There must exist a unique index in the miner's sector set for each // sector ID. The `faults`, `recovered`, and `done` parameters of the // SubmitPoSt method express indices into this sector set. - nssroot, err := AddToSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, params.SectorID, params.CommR, params.CommD) + nssroot, err := AddToSectorSet(ctx, vmctx.Storage(), self.Sectors, params.SectorID, params.CommR, params.CommD) if err != nil { return nil, err } @@ -344,74 +347,48 @@ func (sma StorageMinerActor) GetPower(act *types.Actor, vmctx types.VMContext, p return self.Power.Bytes(), nil } -func SectorIsUnique(cst *hamt.CborIpldStore, sroot cid.Cid, sid types.BigInt) (bool, ActorError) { - nd, err := hamt.LoadNode(context.TODO(), cst, sroot) +func SectorIsUnique(ctx context.Context, s types.Storage, sroot cid.Cid, sid uint64) (bool, ActorError) { + found, _, _, err := GetFromSectorSet(ctx, s, sroot, sid) if err != nil { - return false, aerrors.Escalate(err, "could not load node in HAMT") + return false, err } - if _, err := nd.Find(context.TODO(), sid.String()); err != nil { - if xerrors.Is(err, hamt.ErrNotFound) { - return true, nil - } - return false, aerrors.Escalate(err, "could not find node in HAMT") - } - - return false, nil + return !found, nil } -func AddToSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid, sectorID types.BigInt, commR, commD []byte) (cid.Cid, ActorError) { - nd, err := hamt.LoadNode(ctx, cst, ss) +func AddToSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) { + ssr, err := amt.LoadAMT(types.WrapStorage(s), ss) if err != nil { - return cid.Undef, aerrors.Escalate(err, "could not load HAMT node") + return cid.Undef, aerrors.Escalate(err, "could not load sector set node") } - enc, aerr := SerializeParams([][]byte{commR, commD}) + if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil { + return cid.Undef, aerrors.Escalate(err, "failed to set commitment in sector set") + } + + ncid, err := ssr.Flush() if err != nil { - return cid.Undef, aerrors.Wrap(aerr, "failed to serialize commR and commD for sector set") - } - - if err := nd.Set(ctx, sectorID.String(), enc); err != nil { - return cid.Undef, aerrors.Escalate(err, "failed to set new sector in sector set") - } - - if err := nd.Flush(ctx); err != nil { return cid.Undef, aerrors.Escalate(err, "failed to flush sector set") } - ssroot, err := cst.Put(ctx, nd) - if err != nil { - return cid.Undef, aerrors.Escalate(err, "failed to store new sector set root") - } - - return ssroot, nil + return ncid, nil } -func GetFromSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid, sectorID types.BigInt) (bool, []byte, []byte, ActorError) { - nd, err := hamt.LoadNode(ctx, cst, ss) +func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64) (bool, []byte, []byte, ActorError) { + ssr, err := amt.LoadAMT(types.WrapStorage(s), ss) if err != nil { - return false, nil, nil, aerrors.Escalate(err, "could not load HAMT node") + return false, nil, nil, aerrors.Escalate(err, "could not load sector set node") } - infoIf, err := nd.Find(ctx, sectorID.String()) - if err == hamt.ErrNotFound { - return false, nil, nil, nil - } + var comms [][]byte + err = ssr.Get(sectorID, &comms) if err != nil { + if _, ok := err.(amt.ErrNotFound); ok { + return false, nil, nil, nil + } return false, nil, nil, aerrors.Escalate(err, "failed to find sector in sector set") } - infoB, ok := infoIf.([]byte) - if !ok { - return false, nil, nil, aerrors.Escalate(xerrors.New("casting infoIf to []byte failed"), "") // TODO: Review: how to create aerrror without retcode? - } - - var comms [][]byte // [ [commR], [commD] ] - err = cbor.DecodeInto(infoB, &comms) - if err != nil { - return false, nil, nil, aerrors.Escalate(err, "failed to decode sector set entry") - } - if len(comms) != 2 { return false, nil, nil, aerrors.Escalate(xerrors.New("sector set entry should only have 2 elements"), "") } @@ -420,7 +397,7 @@ func GetFromSectorSet(ctx context.Context, cst *hamt.CborIpldStore, ss cid.Cid, } func ValidatePoRep(maddr address.Address, ssize types.BigInt, params *CommitSectorParams) (bool, ActorError) { - ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorID.Uint64(), params.Proof) + ok, err := sectorbuilder.VerifySeal(ssize.Uint64(), params.CommR, params.CommD, params.CommRStar, maddr, params.SectorID, params.Proof) if err != nil { return false, aerrors.Escalate(err, "verify seal failed") } @@ -548,7 +525,7 @@ type PieceInclVoucherData struct { // TODO: Update spec at https://github.com/fi } type InclusionProof struct { - Sector types.BigInt // for CommD, also verifies the sector is in sector set + Sector uint64 // for CommD, also verifies the sector is in sector set Proof []byte } @@ -574,7 +551,7 @@ func (sma StorageMinerActor) PaymentVerifyInclusion(act *types.Actor, vmctx type return nil, aerrors.Escalate(err, "failed to decode storage payment proof") } - ok, _, commD, aerr := GetFromSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, proof.Sector) + ok, _, commD, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, proof.Sector) if aerr != nil { return nil, aerr } @@ -609,7 +586,7 @@ func (sma StorageMinerActor) PaymentVerifySector(act *types.Actor, vmctx types.V return nil, aerrors.New(1, "unexpected proof bytes") } - ok, _, _, aerr := GetFromSectorSet(context.TODO(), vmctx.Ipld(), self.Sectors, sector) + ok, _, _, aerr := GetFromSectorSet(context.TODO(), vmctx.Storage(), self.Sectors, sector.Uint64()) if aerr != nil { return nil, aerr } diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index dc1b2e126..b43492512 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -1,6 +1,7 @@ package types import ( + "github.com/filecoin-project/go-amt-ipld" "github.com/filecoin-project/go-lotus/chain/actors/aerrors" "github.com/filecoin-project/go-lotus/chain/address" "github.com/ipfs/go-cid" @@ -35,3 +36,28 @@ type VMContext interface { VerifySignature(sig *Signature, from address.Address, data []byte) aerrors.ActorError ChargeGas(uint64) aerrors.ActorError } + +type storageWrapper struct { + s Storage +} + +func (sw *storageWrapper) Put(i interface{}) (cid.Cid, error) { + c, err := sw.s.Put(i) + if err != nil { + return cid.Undef, err + } + + return c, nil +} + +func (sw *storageWrapper) Get(c cid.Cid, out interface{}) error { + if err := sw.s.Get(c, out); err != nil { + return err + } + + return nil +} + +func WrapStorage(s Storage) amt.Blocks { + return &storageWrapper{s} +} diff --git a/go.mod b/go.mod index 7e2508c41..7a6583f40 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.12 require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 github.com/BurntSushi/toml v0.3.1 + github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985 github.com/filecoin-project/go-bls-sigs v0.0.0-20190718224239-4bc4b8a7bbf8 github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 github.com/filecoin-project/go-sectorbuilder v0.0.0-00010101000000-000000000000 @@ -15,12 +16,12 @@ require ( github.com/ipfs/go-blockservice v0.1.2 github.com/ipfs/go-car v0.0.1 github.com/ipfs/go-cid v0.0.3 - github.com/ipfs/go-datastore v0.0.5 + github.com/ipfs/go-datastore v0.1.0 github.com/ipfs/go-ds-badger v0.0.5 github.com/ipfs/go-filestore v0.0.2 github.com/ipfs/go-fs-lock v0.0.1 github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf - github.com/ipfs/go-ipfs-blockstore v0.0.1 + github.com/ipfs/go-ipfs-blockstore v0.1.0 github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 @@ -66,7 +67,7 @@ require ( github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.3.0 - github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804 + github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d github.com/whyrusleeping/sharray v0.0.0-20190718051354-e41931821e33 diff --git a/go.sum b/go.sum index 10669286e..da2034414 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,12 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828222406-5f4c30401bda h1:IZKqXB3VvlRCwroy8GbdHjyxJgueveGPXKiSL8nvI7o= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828222406-5f4c30401bda/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828225523-8bbaf27f2782 h1:J0h4pFlx2ZSm1xWO7e2EesXz5xW/35AMqMzk4VGZ6B8= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828225523-8bbaf27f2782/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985 h1:rpid5Xgp6GnDACqZvugxWvJ8L1HpyttUkzpUk/4BPXk= +github.com/filecoin-project/go-amt-ipld v0.0.0-20190828233303-a41bd0b6f985/go.mod h1:muo8IeR187EUiX5AcMmIb0XSgpSU3qrszgh+pGWf3rY= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543 h1:aMJGfgqe1QDhAVwxRg5fjCRF533xHidiKsugk7Vvzug= github.com/filecoin-project/go-leb128 v0.0.0-20190212224330-8d79a5489543/go.mod h1:mjrHv1cDGJWDlGmC0eDc1E5VJr8DmL9XMUcaFwiuKg8= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -120,6 +126,8 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.1 h1:s7KkiBPfxCeDVo47KySjK0ACPc5GJRUxFpdyWEuDjhw= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= +github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= +github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/go-bitswap v0.0.1/go.mod h1:z+tP3h+HTJ810n1R5yMy2ccKFffJ2F6Vqm/5Bf7vs2c= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= @@ -142,6 +150,8 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -157,6 +167,8 @@ github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf h1:P9Kkd8YCG4 github.com/ipfs/go-hamt-ipld v0.0.12-0.20190822003241-7ff276389cbf/go.mod h1:gaK14QN1GOlYGgq+o+t5+WTExZZogkMt0k0IIBNjXsM= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= +github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= @@ -508,6 +520,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1 h1:wDIXmhgP github.com/whyrusleeping/cbor-gen v0.0.0-20190822002707-4e02357de5c1/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804 h1:wK83hcnZgKf1AyH8804pc4zVZOB2ND+d2cagsuhQmZc= github.com/whyrusleeping/cbor-gen v0.0.0-20190822012446-bb2210dd2804/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a h1:9oEQR9eq2H2JDmglMcrCa+TxUEYy3HKSiNoLIkPNy/U= +github.com/whyrusleeping/cbor-gen v0.0.0-20190822231004-8db835b09a5a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/storage/miner.go b/storage/miner.go index 81e1a35b9..6a4945c57 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,6 +3,7 @@ package storage import ( "context" "fmt" + "github.com/filecoin-project/go-lotus/storage/sector" "github.com/filecoin-project/go-lotus/api" @@ -111,7 +112,7 @@ func (m *Miner) commitSector(ctx context.Context, sinfo sectorbuilder.SectorSeal } params := &actors.CommitSectorParams{ - SectorID: types.NewInt(sinfo.SectorID), + SectorID: sinfo.SectorID, CommD: sinfo.CommD[:], CommR: sinfo.CommR[:], CommRStar: sinfo.CommRStar[:],