From 63ee6b6791b6cb4ebd79ad7c916ea4fe9fdb2f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 4 Feb 2020 07:55:57 +0100 Subject: [PATCH 1/9] remote: Fix put in commit --- cli/chain.go | 2 +- node/impl/storminer.go | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index 8fe34ad16..6f691edee 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -455,7 +455,7 @@ var chainBisectCmd = &cli.Command{ for { mid := (start + end) / 2 - if end - start == 1 { + if end-start == 1 { mid = end start = end } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 7a60119c5..4c54a4074 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -108,11 +108,21 @@ func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Reques } // This is going to get better with worker-to-worker transfers - path, err := sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), id, true) + + path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), id) if err != nil { - log.Error(err) - w.WriteHeader(500) - return + if err != fs.ErrNotFound { + log.Error(err) + w.WriteHeader(500) + return + } + + path, err = sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), id, true) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } } mediatype, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) From 5e6ab42cb868b4e3c2eed32f08edb24217e9a4c5 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 30 Jan 2020 15:48:25 -0800 Subject: [PATCH 2/9] Don't depend on ffi Signed-off-by: Jakub Sztandera --- chain/actors/actor_storagemarket.go | 3 +- chain/actors/actor_storagepower.go | 7 ++- chain/gen/gen.go | 23 ++++--- chain/messagepool/messagepool.go | 5 +- chain/sync.go | 5 +- chain/types/blockheader_cgo.go | 27 -------- chain/types/signature_cgo.go | 60 ------------------ chain/vm/vm.go | 3 +- chain/wallet/wallet.go | 79 +++++------------------ go.sum | 4 ++ lib/sigs/bls/init.go | 52 ++++++++++++++++ lib/sigs/secp/init.go | 58 +++++++++++++++++ lib/sigs/verify.go | 97 +++++++++++++++++++++++++++++ paych/paych.go | 3 +- 14 files changed, 256 insertions(+), 170 deletions(-) delete mode 100644 chain/types/blockheader_cgo.go delete mode 100644 chain/types/signature_cgo.go create mode 100644 lib/sigs/bls/init.go create mode 100644 lib/sigs/secp/init.go create mode 100644 lib/sigs/verify.go diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 8fa4386f7..57911a021 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -18,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" ) type StorageMarketActor struct{} @@ -131,7 +132,7 @@ func (sdp *StorageDealProposal) Verify(worker address.Address) error { return err } - if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil { + if err := sigs.Verify(sdp.ProposerSignature, sdp.Client, buf.Bytes()); err != nil { return err } } diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index af56e0deb..23bd2b779 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -17,6 +17,9 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) type StoragePowerActor struct{} @@ -170,11 +173,11 @@ func (spa StoragePowerActor) ArbitrateConsensusFault(act *types.Actor, vmctx typ return nil, aerrors.Absorb(oerr, 3, "response from 'GetWorkerAddr' was not a valid address") } - if err := params.Block1.CheckBlockSignature(vmctx.Context(), worker); err != nil { + if err := sigs.CheckBlockSignature(params.Block1, vmctx.Context(), worker); err != nil { return nil, aerrors.Absorb(err, 4, "block1 did not have valid signature") } - if err := params.Block2.CheckBlockSignature(vmctx.Context(), worker); err != nil { + if err := sigs.CheckBlockSignature(params.Block2, vmctx.Context(), worker); err != nil { return nil, aerrors.Absorb(err, 5, "block2 did not have valid signature") } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 066ec918a..8486056ff 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -9,34 +9,33 @@ import ( "io/ioutil" "sync/atomic" - "github.com/filecoin-project/lotus/chain/vm" - - ffi "github.com/filecoin-project/filecoin-ffi" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + block "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-car" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" + logging "github.com/ipfs/go-log/v2" "github.com/ipfs/go-merkledag" peer "github.com/libp2p/go-libp2p-core/peer" - "go.opencensus.io/trace" - "golang.org/x/xerrors" + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/cmd/lotus-seed/seed" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" - block "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - logging "github.com/ipfs/go-log/v2" + "go.opencensus.io/trace" + "golang.org/x/xerrors" ) var log = logging.Logger("gen") @@ -631,7 +630,7 @@ func VerifyVRF(ctx context.Context, worker, miner address.Address, p uint64, inp Data: vrfproof, } - if err := sig.Verify(worker, vrfBase); err != nil { + if err := sigs.Verify(sig, worker, vrfBase); err != nil { return xerrors.Errorf("vrf was invalid: %w", err) } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 8a5672dc6..b26427861 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -25,6 +25,9 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -303,7 +306,7 @@ func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet) error return ErrMessageValueTooHigh } - if err := m.Signature.Verify(m.Message.From, m.Message.Cid().Bytes()); err != nil { + if err := sigs.Verify(&m.Signature, m.Message.From, m.Message.Cid().Bytes()); err != nil { log.Warnf("mpooladd signature verification failed: %s", err) return err } diff --git a/chain/sync.go b/chain/sync.go index 7c8e13b36..95ff9cd42 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" ) var log = logging.Logger("chain") @@ -604,7 +605,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } blockSigCheck := async.Err(func() error { - if err := h.CheckBlockSignature(ctx, waddr); err != nil { + if err := sigs.CheckBlockSignature(h, ctx, waddr); err != nil { return xerrors.Errorf("check block signature failed: %w", err) } return nil @@ -787,7 +788,7 @@ func (syncer *Syncer) checkBlockMessages(ctx context.Context, b *types.FullBlock return xerrors.Errorf("failed to resolve key addr: %w", err) } - if err := m.Signature.Verify(kaddr, m.Message.Cid().Bytes()); err != nil { + if err := sigs.Verify(&m.Signature, kaddr, m.Message.Cid().Bytes()); err != nil { return xerrors.Errorf("secpk message %s has invalid signature: %w", m.Cid(), err) } diff --git a/chain/types/blockheader_cgo.go b/chain/types/blockheader_cgo.go deleted file mode 100644 index 653935cbc..000000000 --- a/chain/types/blockheader_cgo.go +++ /dev/null @@ -1,27 +0,0 @@ -//+build cgo - -package types - -import ( - "context" - - "github.com/filecoin-project/go-address" - "go.opencensus.io/trace" - "golang.org/x/xerrors" -) - -func (blk *BlockHeader) CheckBlockSignature(ctx context.Context, worker address.Address) error { - _, span := trace.StartSpan(ctx, "checkBlockSignature") - defer span.End() - - if blk.BlockSig == nil { - return xerrors.New("block signature not present") - } - - sigb, err := blk.SigningBytes() - if err != nil { - return xerrors.Errorf("failed to get block signing bytes: %w", err) - } - - return blk.BlockSig.Verify(worker, sigb) -} diff --git a/chain/types/signature_cgo.go b/chain/types/signature_cgo.go deleted file mode 100644 index a7423d946..000000000 --- a/chain/types/signature_cgo.go +++ /dev/null @@ -1,60 +0,0 @@ -//+build cgo - -package types - -import ( - "fmt" - - bls "github.com/filecoin-project/filecoin-ffi" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-crypto" - "github.com/minio/blake2b-simd" - "golang.org/x/xerrors" -) - -func (s *Signature) Verify(addr address.Address, msg []byte) error { - if s == nil { - return xerrors.Errorf("signature is nil") - } - - if addr.Protocol() == address.ID { - return fmt.Errorf("must resolve ID addresses before using them to verify a signature") - } - b2sum := blake2b.Sum256(msg) - - switch s.Type { - case KTSecp256k1: - pubk, err := crypto.EcRecover(b2sum[:], s.Data) - if err != nil { - return err - } - - maybeaddr, err := address.NewSecp256k1Address(pubk) - if err != nil { - return err - } - - if addr != maybeaddr { - return fmt.Errorf("signature did not match") - } - - return nil - case KTBLS: - digests := []bls.Digest{bls.Hash(bls.Message(msg))} - - var pubk bls.PublicKey - copy(pubk[:], addr.Payload()) - pubkeys := []bls.PublicKey{pubk} - - var sig bls.Signature - copy(sig[:], s.Data) - - if !bls.Verify(&sig, digests, pubkeys) { - return fmt.Errorf("bls signature failed to verify") - } - - return nil - default: - return fmt.Errorf("cannot verify signature of unsupported type: %s", s.Type) - } -} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index cfb9c02e2..e7c09ad04 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/bufbstore" + "github.com/filecoin-project/lotus/lib/sigs" ) var log = logging.Logger("vm") @@ -196,7 +197,7 @@ func (vmctx *VMContext) VerifySignature(sig *types.Signature, act address.Addres act = kaddr } - if err := sig.Verify(act, data); err != nil { + if err := sigs.Verify(sig, act, data); err != nil { return aerrors.New(2, "signature verification failed") } diff --git a/chain/wallet/wallet.go b/chain/wallet/wallet.go index 3a70b506a..3e5d506a8 100644 --- a/chain/wallet/wallet.go +++ b/chain/wallet/wallet.go @@ -2,20 +2,16 @@ package wallet import ( "context" - "fmt" "sort" "strings" "sync" - bls "github.com/filecoin-project/filecoin-ffi" - logging "github.com/ipfs/go-log/v2" - "github.com/minio/blake2b-simd" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-crypto" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" ) var log = logging.Logger("wallet") @@ -61,31 +57,7 @@ func (w *Wallet) Sign(ctx context.Context, addr address.Address, msg []byte) (*t return nil, xerrors.Errorf("signing using key '%s': %w", addr.String(), types.ErrKeyInfoNotFound) } - switch ki.Type { - case types.KTSecp256k1: - b2sum := blake2b.Sum256(msg) - sig, err := crypto.Sign(ki.PrivateKey, b2sum[:]) - if err != nil { - return nil, err - } - - return &types.Signature{ - Type: types.KTSecp256k1, - Data: sig, - }, nil - case types.KTBLS: - var pk bls.PrivateKey - copy(pk[:], ki.PrivateKey) - sig := bls.PrivateKeySign(pk, msg) - - return &types.Signature{ - Type: types.KTBLS, - Data: sig[:], - }, nil - - default: - return nil, fmt.Errorf("cannot sign with unsupported key type: %q", ki.Type) - } + return sigs.Sign(ki.Type, ki.PrivateKey, msg) } func (w *Wallet) findKey(addr address.Address) (*Key, error) { @@ -204,29 +176,15 @@ func (w *Wallet) SetDefault(a address.Address) error { } func GenerateKey(typ string) (*Key, error) { - switch typ { - case types.KTSecp256k1: - priv, err := crypto.GenerateKey() - if err != nil { - return nil, err - } - ki := types.KeyInfo{ - Type: typ, - PrivateKey: priv, - } - - return NewKey(ki) - case types.KTBLS: - priv := bls.PrivateKeyGenerate() - ki := types.KeyInfo{ - Type: typ, - PrivateKey: priv[:], - } - - return NewKey(ki) - default: - return nil, xerrors.Errorf("invalid key type: %s", typ) + pk, err := sigs.Generate(typ) + if err != nil { + return nil, err } + ki := types.KeyInfo{ + Type: typ, + PrivateKey: pk, + } + return NewKey(ki) } func (w *Wallet) GenerateKey(typ string) (address.Address, error) { @@ -277,28 +235,23 @@ func NewKey(keyinfo types.KeyInfo) (*Key, error) { KeyInfo: keyinfo, } + var err error + k.PublicKey, err = sigs.ToPublic(k.Type, k.PrivateKey) + if err != nil { + return nil, err + } + switch k.Type { case types.KTSecp256k1: - k.PublicKey = crypto.PublicKey(k.PrivateKey) - - var err error k.Address, err = address.NewSecp256k1Address(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting Secp256k1 to address: %w", err) } - case types.KTBLS: - var pk bls.PrivateKey - copy(pk[:], k.PrivateKey) - pub := bls.PrivateKeyPublicKey(pk) - k.PublicKey = pub[:] - - var err error k.Address, err = address.NewBLSAddress(k.PublicKey) if err != nil { return nil, xerrors.Errorf("converting BLS to address: %w", err) } - default: return nil, xerrors.Errorf("unknown key type") } diff --git a/go.sum b/go.sum index 29beb8ca5..a56c416d2 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,10 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw= +github.com/filecoin-project/filecoin-ffi v0.0.0-20191204125133-ebb3e13addf1/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= +github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= +github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590 h1:Gwt2HTv1hBWDv+gXVPWfXsQUDCbmJSCtCyLPD5Pz6pg= +github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A= diff --git a/lib/sigs/bls/init.go b/lib/sigs/bls/init.go new file mode 100644 index 000000000..1c22202c8 --- /dev/null +++ b/lib/sigs/bls/init.go @@ -0,0 +1,52 @@ +package bls + +import ( + "fmt" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" +) + +type blsSigner struct{} + +func (blsSigner) GenPrivate() ([]byte, error) { + pk := ffi.PrivateKeyGenerate() + return pk[:], nil +} + +func (blsSigner) ToPublic(priv []byte) ([]byte, error) { + var pk ffi.PrivateKey + copy(pk[:], priv) + pub := ffi.PrivateKeyPublicKey(pk) + return pub[:], nil +} + +func (blsSigner) Sign(p []byte, msg []byte) ([]byte, error) { + var pk ffi.PrivateKey + copy(pk[:], p) + sig := ffi.PrivateKeySign(pk, msg) + return sig[:], nil +} + +func (blsSigner) Verify(sig []byte, a address.Address, msg []byte) error { + digests := []ffi.Digest{ffi.Hash(ffi.Message(msg))} + + var pubk ffi.PublicKey + copy(pubk[:], a.Payload()) + pubkeys := []ffi.PublicKey{pubk} + + var s ffi.Signature + copy(s[:], sig) + + if !ffi.Verify(&s, digests, pubkeys) { + return fmt.Errorf("bls signature failed to verify") + } + + return nil +} + +func init() { + sigs.RegisterSignature(types.KTBLS, blsSigner{}) +} diff --git a/lib/sigs/secp/init.go b/lib/sigs/secp/init.go new file mode 100644 index 000000000..84680ac81 --- /dev/null +++ b/lib/sigs/secp/init.go @@ -0,0 +1,58 @@ +package secp + +import ( + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-crypto" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" + "github.com/minio/blake2b-simd" +) + +type secpSigner struct{} + +func (secpSigner) GenPrivate() ([]byte, error) { + priv, err := crypto.GenerateKey() + if err != nil { + return nil, err + } + return priv, nil +} + +func (secpSigner) ToPublic(pk []byte) ([]byte, error) { + return crypto.PublicKey(pk), nil +} + +func (secpSigner) Sign(pk []byte, msg []byte) ([]byte, error) { + b2sum := blake2b.Sum256(msg) + sig, err := crypto.Sign(pk, b2sum[:]) + if err != nil { + return nil, err + } + + return sig, nil +} + +func (secpSigner) Verify(sig []byte, a address.Address, msg []byte) error { + b2sum := blake2b.Sum256(msg) + pubk, err := crypto.EcRecover(b2sum[:], sig) + if err != nil { + return err + } + + maybeaddr, err := address.NewSecp256k1Address(pubk) + if err != nil { + return err + } + + if a != maybeaddr { + return fmt.Errorf("signature did not match") + } + + return nil +} + +func init() { + sigs.RegisterSignature(types.KTSecp256k1, secpSigner{}) +} diff --git a/lib/sigs/verify.go b/lib/sigs/verify.go new file mode 100644 index 000000000..7a8a56ecb --- /dev/null +++ b/lib/sigs/verify.go @@ -0,0 +1,97 @@ +package sigs + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "go.opencensus.io/trace" + "golang.org/x/xerrors" +) + +type SigShim interface { + GenPrivate() ([]byte, error) + ToPublic(pk []byte) ([]byte, error) + Sign(pk []byte, msg []byte) ([]byte, error) + Verify(sig []byte, a address.Address, msg []byte) error +} + +var sigs map[string]SigShim + +// RegisterSig should be only used during init +func RegisterSignature(name string, vs SigShim) { + if sigs == nil { + sigs = make(map[string]SigShim) + } + sigs[name] = vs +} + +func Sign(sigType string, privkey []byte, msg []byte) (*types.Signature, error) { + sv, ok := sigs[sigType] + if !ok { + return nil, fmt.Errorf("cannot sign message with signature of unsupported type: %s", sigType) + } + + sb, err := sv.Sign(privkey, msg) + if err != nil { + return nil, err + } + return &types.Signature{ + Type: sigType, + Data: sb, + }, nil +} + +func Verify(sig *types.Signature, addr address.Address, msg []byte) error { + if sig == nil { + return xerrors.Errorf("signature is nil") + } + + if addr.Protocol() == address.ID { + return fmt.Errorf("must resolve ID addresses before using them to verify a signature") + } + + sv, ok := sigs[sig.Type] + if !ok { + return fmt.Errorf("cannot verify signature of unsupported type: %s", sig.Type) + } + + return sv.Verify(sig.Data, addr, msg) +} + +func Generate(sigType string) ([]byte, error) { + sv, ok := sigs[sigType] + if !ok { + return nil, fmt.Errorf("cannot generate private key of unsupported type: %s", sigType) + } + + return sv.GenPrivate() +} + +func ToPublic(sigType string, pk []byte) ([]byte, error) { + sv, ok := sigs[sigType] + if !ok { + return nil, fmt.Errorf("cannot generate public key of unsupported type: %s", sigType) + } + + return sv.ToPublic(pk) +} + +func CheckBlockSignature(blk *types.BlockHeader, ctx context.Context, worker address.Address) error { + _, span := trace.StartSpan(ctx, "checkBlockSignature") + defer span.End() + + if blk.BlockSig == nil { + return xerrors.New("block signature not present") + } + + sigb, err := blk.SigningBytes() + if err != nil { + return xerrors.Errorf("failed to get block signing bytes: %w", err) + } + + _ = sigb + //return blk.BlockSig.Verify(worker, sigb) + return nil +} diff --git a/paych/paych.go b/paych/paych.go index 28f4c0fca..375fe7395 100644 --- a/paych/paych.go +++ b/paych/paych.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/impl/full" ) @@ -138,7 +139,7 @@ func (pm *Manager) CheckVoucherValid(ctx context.Context, ch address.Address, sv // TODO: technically, either party may create and sign a voucher. // However, for now, we only accept them from the channel creator. // More complex handling logic can be added later - if err := sv.Signature.Verify(pca.From, vb); err != nil { + if err := sigs.Verify(sv.Signature, pca.From, vb); err != nil { return err } From ad9e4db3d68c1a69b5436f1f6a26e7b9dc04ea22 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 30 Jan 2020 16:20:04 -0800 Subject: [PATCH 3/9] Reduce sig import locations Signed-off-by: Jakub Sztandera --- chain/actors/actor_storagepower.go | 2 -- chain/actors/actor_storagepower_test.go | 2 ++ chain/messagepool/messagepool.go | 2 -- chain/messagepool/messagepool_test.go | 2 ++ node/builder.go | 2 ++ 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chain/actors/actor_storagepower.go b/chain/actors/actor_storagepower.go index 23bd2b779..1a2491aa7 100644 --- a/chain/actors/actor_storagepower.go +++ b/chain/actors/actor_storagepower.go @@ -18,8 +18,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/aerrors" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" - _ "github.com/filecoin-project/lotus/lib/sigs/bls" - _ "github.com/filecoin-project/lotus/lib/sigs/secp" ) type StoragePowerActor struct{} diff --git a/chain/actors/actor_storagepower_test.go b/chain/actors/actor_storagepower_test.go index c84492eae..51c3e0247 100644 --- a/chain/actors/actor_storagepower_test.go +++ b/chain/actors/actor_storagepower_test.go @@ -13,6 +13,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" cid "github.com/ipfs/go-cid" hamt "github.com/ipfs/go-hamt-ipld" diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index b26427861..5a9deb569 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -26,8 +26,6 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/sigs" - _ "github.com/filecoin-project/lotus/lib/sigs/bls" - _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/node/modules/dtypes" ) diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index 514806eb1..e0b1022ab 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -9,6 +9,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/mock" "github.com/filecoin-project/lotus/chain/wallet" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ) diff --git a/node/builder.go b/node/builder.go index 028778c09..d7fe6b1be 100644 --- a/node/builder.go +++ b/node/builder.go @@ -37,6 +37,8 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/chain/wallet" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" "github.com/filecoin-project/lotus/markets/storageadapter" "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/config" From cb05001e5a9e8d9076126ced5572b4b5b49c9f7a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 30 Jan 2020 16:38:03 -0800 Subject: [PATCH 4/9] mod tidy Signed-off-by: Jakub Sztandera --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index a56c416d2..29beb8ca5 100644 --- a/go.sum +++ b/go.sum @@ -97,10 +97,6 @@ github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGj github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuYCW0wUb26AvnD4WK59M= github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw= -github.com/filecoin-project/filecoin-ffi v0.0.0-20191204125133-ebb3e13addf1/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= -github.com/filecoin-project/filecoin-ffi v0.0.0-20191213130254-f261762ff8ed/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= -github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590 h1:Gwt2HTv1hBWDv+gXVPWfXsQUDCbmJSCtCyLPD5Pz6pg= -github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590/go.mod h1:yA6YM1jzYoKaPrFjJIKV3ZmcGZJlo8rpmxfzBRNlLwM= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g= github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms= github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A= From 5bc1cb2aa3c220471fab385b10f83cc1077c506d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 4 Feb 2020 07:24:25 +0100 Subject: [PATCH 5/9] Fix CheckBlockSignature, add docks Signed-off-by: Jakub Sztandera --- lib/sigs/doc.go | 9 +++++++ lib/sigs/{verify.go => sigs.go} | 43 ++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 lib/sigs/doc.go rename lib/sigs/{verify.go => sigs.go} (85%) diff --git a/lib/sigs/doc.go b/lib/sigs/doc.go new file mode 100644 index 000000000..637cd2bcd --- /dev/null +++ b/lib/sigs/doc.go @@ -0,0 +1,9 @@ +// Sigs package allows for signing, verifying signatures and key generation +// using key types selected by package user. +// +// For support of secp256k1 import: +// _ "github.com/filecoin-project/lotus/lib/sigs/secp" +// +// For support of Filecoin BLS import: +// _ "github.com/filecoin-project/lotus/lib/sigs/bls" +package sigs diff --git a/lib/sigs/verify.go b/lib/sigs/sigs.go similarity index 85% rename from lib/sigs/verify.go rename to lib/sigs/sigs.go index 7a8a56ecb..9d1bc4b6d 100644 --- a/lib/sigs/verify.go +++ b/lib/sigs/sigs.go @@ -10,23 +10,8 @@ import ( "golang.org/x/xerrors" ) -type SigShim interface { - GenPrivate() ([]byte, error) - ToPublic(pk []byte) ([]byte, error) - Sign(pk []byte, msg []byte) ([]byte, error) - Verify(sig []byte, a address.Address, msg []byte) error -} - -var sigs map[string]SigShim - -// RegisterSig should be only used during init -func RegisterSignature(name string, vs SigShim) { - if sigs == nil { - sigs = make(map[string]SigShim) - } - sigs[name] = vs -} - +// Sign takes in signature type, private key and message. Returns a signature for that message. +// Valid sigTypes are: "secp256k1" and "bls" func Sign(sigType string, privkey []byte, msg []byte) (*types.Signature, error) { sv, ok := sigs[sigType] if !ok { @@ -43,6 +28,7 @@ func Sign(sigType string, privkey []byte, msg []byte) (*types.Signature, error) }, nil } +// Verify verifies signatures func Verify(sig *types.Signature, addr address.Address, msg []byte) error { if sig == nil { return xerrors.Errorf("signature is nil") @@ -60,6 +46,7 @@ func Verify(sig *types.Signature, addr address.Address, msg []byte) error { return sv.Verify(sig.Data, addr, msg) } +// Generate generates private key of given type func Generate(sigType string) ([]byte, error) { sv, ok := sigs[sigType] if !ok { @@ -69,6 +56,7 @@ func Generate(sigType string) ([]byte, error) { return sv.GenPrivate() } +// ToPublic converts private key to public key func ToPublic(sigType string, pk []byte) ([]byte, error) { sv, ok := sigs[sigType] if !ok { @@ -92,6 +80,23 @@ func CheckBlockSignature(blk *types.BlockHeader, ctx context.Context, worker add } _ = sigb - //return blk.BlockSig.Verify(worker, sigb) - return nil + return Verify(blk.BlockSig, worker, sigb) +} + +// SigShim is used for introducing signature functions +type SigShim interface { + GenPrivate() ([]byte, error) + ToPublic(pk []byte) ([]byte, error) + Sign(pk []byte, msg []byte) ([]byte, error) + Verify(sig []byte, a address.Address, msg []byte) error +} + +var sigs map[string]SigShim + +// RegisterSig should be only used during init +func RegisterSignature(name string, vs SigShim) { + if sigs == nil { + sigs = make(map[string]SigShim) + } + sigs[name] = vs } From 1a9c77540777b08480e134323922403862849aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 4 Feb 2020 20:04:49 +0100 Subject: [PATCH 6/9] worker: Prefetch data in background --- cmd/lotus-seal-worker/main.go | 54 ++++++++++++++++++++++++++++++- cmd/lotus-seal-worker/sub.go | 36 ++++++++------------- cmd/lotus-seal-worker/transfer.go | 10 ++++++ 3 files changed, 76 insertions(+), 24 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 6170f8239..4bcd00c80 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -2,7 +2,9 @@ package main import ( "os" + "sync" + "github.com/filecoin-project/go-sectorbuilder" "github.com/mitchellh/go-homedir" logging "github.com/ipfs/go-log/v2" @@ -19,6 +21,11 @@ import ( var log = logging.Logger("main") +const ( + workers = 1 // TODO: Configurability + transfers = 1 +) + func main() { lotuslog.SetupLogLevels() @@ -67,6 +74,11 @@ func main() { } } +type limits struct { + workLimit chan struct{} + transferLimit chan struct{} +} + var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", @@ -106,6 +118,46 @@ var runCmd = &cli.Command{ log.Warn("Shutting down..") }() - return acceptJobs(ctx, nodeApi, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")) + limiter := &limits{ + workLimit: make(chan struct{}, workers), + transferLimit: make(chan struct{}, transfers), + } + + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } + + sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ + SectorSize: ssize, + Miner: act, + WorkerThreads: workers, + Paths: sectorbuilder.SimplePath(r), + }) + if err != nil { + return err + } + + nQueues := workers + transfers + var wg sync.WaitGroup + wg.Add(nQueues) + + for i := 0; i < nQueues; i++ { + go func() { + defer wg.Done() + + if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { + log.Warnf("%+v", err) + return + } + }() + } + + wg.Wait() + return nil }, } diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index ec5d88870..e0b8b0705 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -18,30 +18,12 @@ type worker struct { repo string auth http.Header - sb *sectorbuilder.SectorBuilder + limiter *limits + sb *sectorbuilder.SectorBuilder } -func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, auth http.Header, repo string, noprecommit, nocommit bool) error { - act, err := api.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := api.ActorSectorSize(ctx, act) - if err != nil { - return err - } - - sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ - SectorSize: ssize, - Miner: act, - WorkerThreads: 1, - Paths: sectorbuilder.SimplePath(repo), - }) - if err != nil { - return err - } - - if err := paramfetch.GetParams(build.ParametersJson(), ssize); err != nil { +func acceptJobs(ctx context.Context, api lapi.StorageMiner, sb *sectorbuilder.SectorBuilder, limiter *limits, endpoint string, auth http.Header, repo string, noprecommit, nocommit bool) error { + if err := paramfetch.GetParams(build.ParametersJson(), sb.SectorSize()); err != nil { return xerrors.Errorf("get params: %w", err) } @@ -50,7 +32,9 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut minerEndpoint: endpoint, auth: auth, repo: repo, - sb: sb, + + limiter: limiter, + sb: sb, } tasks, err := api.WorkerQueue(ctx, sectorbuilder.WorkerCfg{ @@ -103,7 +87,10 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) switch task.Type { case sectorbuilder.WorkerPreCommit: + w.limiter.workLimit <- struct{}{} rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces) + <-w.limiter.workLimit + if err != nil { return errRes(xerrors.Errorf("precomitting: %w", err)) } @@ -121,7 +108,10 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) return errRes(xerrors.Errorf("cleaning up staged sector: %w", err)) } case sectorbuilder.WorkerCommit: + w.limiter.workLimit <- struct{}{} proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco) + <-w.limiter.workLimit + if err != nil { return errRes(xerrors.Errorf("comitting: %w", err)) } diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go index e9e8e760f..6091a628f 100644 --- a/cmd/lotus-seal-worker/transfer.go +++ b/cmd/lotus-seal-worker/transfer.go @@ -78,6 +78,11 @@ func (w *worker) fetch(typ string, sectorID uint64) error { } func (w *worker) push(typ string, sectorID uint64) error { + w.limiter.transferLimit <- struct{}{} + defer func() { + <-w.limiter.transferLimit + }() + filename, err := w.sb.SectorPath(fs.DataType(typ), sectorID) if err != nil { return err @@ -147,6 +152,11 @@ func (w *worker) remove(typ string, sectorID uint64) error { } func (w *worker) fetchSector(sectorID uint64, typ sectorbuilder.WorkerTaskType) error { + w.limiter.transferLimit <- struct{}{} + defer func() { + <-w.limiter.transferLimit + }() + var err error switch typ { case sectorbuilder.WorkerPreCommit: From 63af9f2a9edc4836cb90fb1f3ef1421efc7a4650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 4 Feb 2020 20:17:18 +0100 Subject: [PATCH 7/9] worker: Paramfetch once --- cmd/lotus-seal-worker/main.go | 8 +++++++- cmd/lotus-seal-worker/sub.go | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 4bcd00c80..208d64eb5 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -4,6 +4,7 @@ import ( "os" "sync" + paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-sectorbuilder" "github.com/mitchellh/go-homedir" @@ -11,12 +12,13 @@ import ( "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + manet "github.com/multiformats/go-multiaddr-net" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" - manet "github.com/multiformats/go-multiaddr-net" ) var log = logging.Logger("main") @@ -132,6 +134,10 @@ var runCmd = &cli.Command{ return err } + if err := paramfetch.GetParams(build.ParametersJson(), ssize); err != nil { + return xerrors.Errorf("get params: %w", err) + } + sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ SectorSize: ssize, Miner: act, diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go index e0b8b0705..0c2d233de 100644 --- a/cmd/lotus-seal-worker/sub.go +++ b/cmd/lotus-seal-worker/sub.go @@ -4,12 +4,10 @@ import ( "context" "net/http" - paramfetch "github.com/filecoin-project/go-paramfetch" "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/build" ) type worker struct { @@ -23,10 +21,6 @@ type worker struct { } func acceptJobs(ctx context.Context, api lapi.StorageMiner, sb *sectorbuilder.SectorBuilder, limiter *limits, endpoint string, auth http.Header, repo string, noprecommit, nocommit bool) error { - if err := paramfetch.GetParams(build.ParametersJson(), sb.SectorSize()); err != nil { - return xerrors.Errorf("get params: %w", err) - } - w := &worker{ api: api, minerEndpoint: endpoint, From 4bd6d7be1a39d8a98c7cae2bbf00df37ad9cc3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 4 Feb 2020 21:46:31 +0100 Subject: [PATCH 8/9] storageminer: Work around broken fastPledgeCommitment --- storage/sealing/garbage.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index 3274f6aec..ca8c1e454 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -7,7 +7,6 @@ import ( "math" "math/bits" "math/rand" - "runtime" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" @@ -41,7 +40,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { - commP, err := m.fastPledgeCommitment(size, uint64(runtime.NumCPU())) + commP, err := m.fastPledgeCommitment(size, uint64(1)) if err != nil { return nil, err } @@ -101,7 +100,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie out := make([]Piece, len(sizes)) for i, size := range sizes { - ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(runtime.NumCPU())), existingPieceSizes) + ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(1)), existingPieceSizes) if err != nil { return nil, xerrors.Errorf("add piece: %w", err) } From 70da3e124a86df8b815ce7b0475954f0bb549835 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Tue, 4 Feb 2020 23:49:10 +0100 Subject: [PATCH 9/9] Fix lotus-seed with sigs changes Signed-off-by: Jakub Sztandera --- cmd/lotus-seed/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/lotus-seed/main.go b/cmd/lotus-seed/main.go index b30f523c9..f6e351508 100644 --- a/cmd/lotus-seed/main.go +++ b/cmd/lotus-seed/main.go @@ -8,6 +8,9 @@ import ( "encoding/json" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + _ "github.com/filecoin-project/lotus/lib/sigs/bls" + _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" badger "github.com/ipfs/go-ds-badger2"