handle rebase
This commit is contained in:
laser
parent
4bbc693d9c
commit
20321bb476
@ -9,10 +9,10 @@ import (
|
|||||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
ffi "github.com/filecoin-project/filecoin-ffi"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/go-amt-ipld"
|
"github.com/filecoin-project/go-amt-ipld"
|
||||||
"github.com/ipfs/go-cid"
|
"github.com/ipfs/go-cid"
|
||||||
|
|||||||
@ -7,12 +7,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
||||||
"github.com/filecoin-project/lotus/chain/stmgr"
|
"github.com/filecoin-project/lotus/chain/stmgr"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
hamt "github.com/ipfs/go-hamt-ipld"
|
hamt "github.com/ipfs/go-hamt-ipld"
|
||||||
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
blockstore "github.com/ipfs/go-ipfs-blockstore"
|
||||||
cbg "github.com/whyrusleeping/cbor-gen"
|
cbg "github.com/whyrusleeping/cbor-gen"
|
||||||
|
|||||||
@ -14,10 +14,10 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
"github.com/filecoin-project/lotus/chain/actors/aerrors"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type StorageMarketActor struct{}
|
type StorageMarketActor struct{}
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
"github.com/filecoin-project/lotus/chain/events"
|
"github.com/filecoin-project/lotus/chain/events"
|
||||||
@ -20,7 +21,6 @@ import (
|
|||||||
"github.com/filecoin-project/lotus/chain/store"
|
"github.com/filecoin-project/lotus/chain/store"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/chain/wallet"
|
"github.com/filecoin-project/lotus/chain/wallet"
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
"github.com/filecoin-project/lotus/node/impl/full"
|
"github.com/filecoin-project/lotus/node/impl/full"
|
||||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||||
"github.com/filecoin-project/lotus/retrieval/discovery"
|
"github.com/filecoin-project/lotus/retrieval/discovery"
|
||||||
|
|||||||
@ -14,11 +14,11 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/datatransfer"
|
"github.com/filecoin-project/lotus/datatransfer"
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||||
"github.com/filecoin-project/lotus/storage"
|
"github.com/filecoin-project/lotus/storage"
|
||||||
"github.com/filecoin-project/lotus/storage/sectorblocks"
|
"github.com/filecoin-project/lotus/storage/sectorblocks"
|
||||||
|
|||||||
@ -12,9 +12,9 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
|
|
||||||
"github.com/ipfs/go-cid"
|
"github.com/ipfs/go-cid"
|
||||||
inet "github.com/libp2p/go-libp2p-core/network"
|
inet "github.com/libp2p/go-libp2p-core/network"
|
||||||
|
|||||||
@ -15,11 +15,11 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
"github.com/filecoin-project/lotus/chain/deals"
|
"github.com/filecoin-project/lotus/chain/deals"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var blockGenerator = blocksutil.NewBlockGenerator()
|
var blockGenerator = blocksutil.NewBlockGenerator()
|
||||||
|
|||||||
@ -23,10 +23,10 @@ import (
|
|||||||
"gopkg.in/urfave/cli.v2"
|
"gopkg.in/urfave/cli.v2"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/genesis"
|
"github.com/filecoin-project/lotus/genesis"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var log = logging.Logger("lotus-bench")
|
var log = logging.Logger("lotus-bench")
|
||||||
|
|||||||
@ -7,8 +7,8 @@ import (
|
|||||||
paramfetch "github.com/filecoin-project/go-paramfetch"
|
paramfetch "github.com/filecoin-project/go-paramfetch"
|
||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
|
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
lapi "github.com/filecoin-project/lotus/api"
|
lapi "github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type worker struct {
|
type worker struct {
|
||||||
|
|||||||
@ -22,6 +22,7 @@ import (
|
|||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
"github.com/filecoin-project/go-cbor-util"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
lapi "github.com/filecoin-project/lotus/api"
|
lapi "github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/actors"
|
"github.com/filecoin-project/lotus/chain/actors"
|
||||||
@ -29,7 +30,6 @@ import (
|
|||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
lcli "github.com/filecoin-project/lotus/cli"
|
lcli "github.com/filecoin-project/lotus/cli"
|
||||||
"github.com/filecoin-project/lotus/genesis"
|
"github.com/filecoin-project/lotus/genesis"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
"github.com/filecoin-project/lotus/miner"
|
"github.com/filecoin-project/lotus/miner"
|
||||||
"github.com/filecoin-project/lotus/node/modules"
|
"github.com/filecoin-project/lotus/node/modules"
|
||||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||||
|
|||||||
1
go.mod
1
go.mod
@ -82,6 +82,7 @@ require (
|
|||||||
github.com/onsi/ginkgo v1.9.0 // indirect
|
github.com/onsi/ginkgo v1.9.0 // indirect
|
||||||
github.com/onsi/gomega v1.6.0 // indirect
|
github.com/onsi/gomega v1.6.0 // indirect
|
||||||
github.com/opentracing/opentracing-go v1.1.0
|
github.com/opentracing/opentracing-go v1.1.0
|
||||||
|
github.com/otiai10/copy v1.0.2
|
||||||
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
|
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
|
||||||
github.com/stretchr/testify v1.4.0
|
github.com/stretchr/testify v1.4.0
|
||||||
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
|
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
|
||||||
|
|||||||
12
go.sum
12
go.sum
@ -89,19 +89,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a
|
|||||||
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
|
github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
|
||||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
|
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
|
||||||
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
|
||||||
<<<<<<< HEAD
|
|
||||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
|
||||||
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
|
||||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY=
|
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY=
|
||||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
|
github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
|
||||||
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
|
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
|
||||||
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
||||||
=======
|
|
||||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20191220204520-82965a74eaca h1:qe8gfAQ+WNBh2MCRN9oRN3f+eGC+p8jfOhLhNmcLGxc=
|
|
||||||
github.com/filecoin-project/go-sectorbuilder v0.0.0-20191220204520-82965a74eaca/go.mod h1:tOSdS//OQy+/03Fe9JDGtfJX/iTQIlYig41ZzKh5cwU=
|
|
||||||
github.com/filecoin-project/go-statestore v0.0.0-20191220165813-c30fad612418 h1:6qIy+u0HRpRz7iisUTRxYWhTbAES1kt8R/wn7hCybrU=
|
|
||||||
github.com/filecoin-project/go-statestore v0.0.0-20191220165813-c30fad612418/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
|
|
||||||
>>>>>>> go mod tidy
|
|
||||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
|
github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
|
||||||
@ -490,11 +483,6 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
|
|||||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||||
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
||||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||||
<<<<<<< HEAD
|
|
||||||
=======
|
|
||||||
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
|
|
||||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
|
||||||
>>>>>>> go mod tidy
|
|
||||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||||
|
|||||||
@ -1,799 +0,0 @@
|
|||||||
package sectorbuilder
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
|
|
||||||
sectorbuilder "github.com/filecoin-project/filecoin-ffi"
|
|
||||||
"github.com/ipfs/go-datastore"
|
|
||||||
logging "github.com/ipfs/go-log"
|
|
||||||
dcopy "github.com/otiai10/copy"
|
|
||||||
"golang.org/x/xerrors"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
|
||||||
"github.com/filecoin-project/lotus/build"
|
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
|
||||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
|
||||||
)
|
|
||||||
|
|
||||||
const PoStReservedWorkers = 1
|
|
||||||
const PoRepProofPartitions = 10
|
|
||||||
|
|
||||||
var lastSectorIdKey = datastore.NewKey("/sectorbuilder/last")
|
|
||||||
|
|
||||||
var log = logging.Logger("sectorbuilder")
|
|
||||||
|
|
||||||
type SortedPublicSectorInfo = sectorbuilder.SortedPublicSectorInfo
|
|
||||||
type SortedPrivateSectorInfo = sectorbuilder.SortedPrivateSectorInfo
|
|
||||||
|
|
||||||
type SealTicket = sectorbuilder.SealTicket
|
|
||||||
|
|
||||||
type SealSeed = sectorbuilder.SealSeed
|
|
||||||
|
|
||||||
type SealPreCommitOutput = sectorbuilder.SealPreCommitOutput
|
|
||||||
|
|
||||||
type SealCommitOutput = sectorbuilder.SealCommitOutput
|
|
||||||
|
|
||||||
type PublicPieceInfo = sectorbuilder.PublicPieceInfo
|
|
||||||
|
|
||||||
type RawSealPreCommitOutput sectorbuilder.RawSealPreCommitOutput
|
|
||||||
|
|
||||||
type EPostCandidate = sectorbuilder.Candidate
|
|
||||||
|
|
||||||
const CommLen = sectorbuilder.CommitmentBytesLen
|
|
||||||
|
|
||||||
type WorkerCfg struct {
|
|
||||||
NoPreCommit bool
|
|
||||||
NoCommit bool
|
|
||||||
|
|
||||||
// TODO: 'cost' info, probably in terms of sealing + transfer speed
|
|
||||||
}
|
|
||||||
|
|
||||||
type SectorBuilder struct {
|
|
||||||
ds dtypes.MetadataDS
|
|
||||||
idLk sync.Mutex
|
|
||||||
|
|
||||||
ssize uint64
|
|
||||||
lastID uint64
|
|
||||||
|
|
||||||
Miner address.Address
|
|
||||||
|
|
||||||
unsealLk sync.Mutex
|
|
||||||
|
|
||||||
noCommit bool
|
|
||||||
noPreCommit bool
|
|
||||||
rateLimit chan struct{}
|
|
||||||
|
|
||||||
precommitTasks chan workerCall
|
|
||||||
commitTasks chan workerCall
|
|
||||||
|
|
||||||
taskCtr uint64
|
|
||||||
remoteLk sync.Mutex
|
|
||||||
remoteCtr int
|
|
||||||
remotes map[int]*remote
|
|
||||||
remoteResults map[uint64]chan<- SealRes
|
|
||||||
|
|
||||||
addPieceWait int32
|
|
||||||
preCommitWait int32
|
|
||||||
commitWait int32
|
|
||||||
unsealWait int32
|
|
||||||
|
|
||||||
fsLk sync.Mutex
|
|
||||||
filesystem *fs // TODO: multi-fs support
|
|
||||||
|
|
||||||
stopping chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
type JsonRSPCO struct {
|
|
||||||
CommD []byte
|
|
||||||
CommR []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rspco *RawSealPreCommitOutput) ToJson() JsonRSPCO {
|
|
||||||
return JsonRSPCO{
|
|
||||||
CommD: rspco.CommD[:],
|
|
||||||
CommR: rspco.CommR[:],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rspco *JsonRSPCO) rspco() RawSealPreCommitOutput {
|
|
||||||
var out RawSealPreCommitOutput
|
|
||||||
copy(out.CommD[:], rspco.CommD)
|
|
||||||
copy(out.CommR[:], rspco.CommR)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
type SealRes struct {
|
|
||||||
Err string
|
|
||||||
GoErr error `json:"-"`
|
|
||||||
|
|
||||||
Proof []byte
|
|
||||||
Rspco JsonRSPCO
|
|
||||||
}
|
|
||||||
|
|
||||||
type remote struct {
|
|
||||||
lk sync.Mutex
|
|
||||||
|
|
||||||
sealTasks chan<- WorkerTask
|
|
||||||
busy uint64 // only for metrics
|
|
||||||
}
|
|
||||||
|
|
||||||
type Config struct {
|
|
||||||
SectorSize uint64
|
|
||||||
Miner address.Address
|
|
||||||
|
|
||||||
WorkerThreads uint8
|
|
||||||
FallbackLastID uint64
|
|
||||||
NoCommit bool
|
|
||||||
NoPreCommit bool
|
|
||||||
|
|
||||||
Dir string
|
|
||||||
_ struct{} // guard against nameless init
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(cfg *Config, ds dtypes.MetadataDS) (*SectorBuilder, error) {
|
|
||||||
if cfg.WorkerThreads < PoStReservedWorkers {
|
|
||||||
return nil, xerrors.Errorf("minimum worker threads is %d, specified %d", PoStReservedWorkers, cfg.WorkerThreads)
|
|
||||||
}
|
|
||||||
|
|
||||||
var lastUsedID uint64
|
|
||||||
b, err := ds.Get(lastSectorIdKey)
|
|
||||||
switch err {
|
|
||||||
case nil:
|
|
||||||
i, err := strconv.ParseInt(string(b), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
lastUsedID = uint64(i)
|
|
||||||
case datastore.ErrNotFound:
|
|
||||||
lastUsedID = cfg.FallbackLastID
|
|
||||||
default:
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
rlimit := cfg.WorkerThreads - PoStReservedWorkers
|
|
||||||
|
|
||||||
sealLocal := rlimit > 0
|
|
||||||
|
|
||||||
if rlimit == 0 {
|
|
||||||
rlimit = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
sb := &SectorBuilder{
|
|
||||||
ds: ds,
|
|
||||||
|
|
||||||
ssize: cfg.SectorSize,
|
|
||||||
lastID: lastUsedID,
|
|
||||||
|
|
||||||
filesystem: openFs(cfg.Dir),
|
|
||||||
|
|
||||||
Miner: cfg.Miner,
|
|
||||||
|
|
||||||
noPreCommit: cfg.NoPreCommit || !sealLocal,
|
|
||||||
noCommit: cfg.NoCommit || !sealLocal,
|
|
||||||
rateLimit: make(chan struct{}, rlimit),
|
|
||||||
|
|
||||||
taskCtr: 1,
|
|
||||||
precommitTasks: make(chan workerCall),
|
|
||||||
commitTasks: make(chan workerCall),
|
|
||||||
remoteResults: map[uint64]chan<- SealRes{},
|
|
||||||
remotes: map[int]*remote{},
|
|
||||||
|
|
||||||
stopping: make(chan struct{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := sb.filesystem.init(); err != nil {
|
|
||||||
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return sb, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewStandalone(cfg *Config) (*SectorBuilder, error) {
|
|
||||||
sb := &SectorBuilder{
|
|
||||||
ds: nil,
|
|
||||||
|
|
||||||
ssize: cfg.SectorSize,
|
|
||||||
|
|
||||||
Miner: cfg.Miner,
|
|
||||||
filesystem: openFs(cfg.Dir),
|
|
||||||
|
|
||||||
taskCtr: 1,
|
|
||||||
remotes: map[int]*remote{},
|
|
||||||
rateLimit: make(chan struct{}, cfg.WorkerThreads),
|
|
||||||
stopping: make(chan struct{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := sb.filesystem.init(); err != nil {
|
|
||||||
return nil, xerrors.Errorf("initializing sectorbuilder filesystem: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return sb, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) checkRateLimit() {
|
|
||||||
if cap(sb.rateLimit) == len(sb.rateLimit) {
|
|
||||||
log.Warn("rate-limiting local sectorbuilder call")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) RateLimit() func() {
|
|
||||||
sb.checkRateLimit()
|
|
||||||
|
|
||||||
sb.rateLimit <- struct{}{}
|
|
||||||
|
|
||||||
return func() {
|
|
||||||
<-sb.rateLimit
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type WorkerStats struct {
|
|
||||||
LocalFree int
|
|
||||||
LocalReserved int
|
|
||||||
LocalTotal int
|
|
||||||
// todo: post in progress
|
|
||||||
RemotesTotal int
|
|
||||||
RemotesFree int
|
|
||||||
|
|
||||||
AddPieceWait int
|
|
||||||
PreCommitWait int
|
|
||||||
CommitWait int
|
|
||||||
UnsealWait int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) WorkerStats() WorkerStats {
|
|
||||||
sb.remoteLk.Lock()
|
|
||||||
defer sb.remoteLk.Unlock()
|
|
||||||
|
|
||||||
remoteFree := len(sb.remotes)
|
|
||||||
for _, r := range sb.remotes {
|
|
||||||
if r.busy > 0 {
|
|
||||||
remoteFree--
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return WorkerStats{
|
|
||||||
LocalFree: cap(sb.rateLimit) - len(sb.rateLimit),
|
|
||||||
LocalReserved: PoStReservedWorkers,
|
|
||||||
LocalTotal: cap(sb.rateLimit) + PoStReservedWorkers,
|
|
||||||
RemotesTotal: len(sb.remotes),
|
|
||||||
RemotesFree: remoteFree,
|
|
||||||
|
|
||||||
AddPieceWait: int(atomic.LoadInt32(&sb.addPieceWait)),
|
|
||||||
PreCommitWait: int(atomic.LoadInt32(&sb.preCommitWait)),
|
|
||||||
CommitWait: int(atomic.LoadInt32(&sb.commitWait)),
|
|
||||||
UnsealWait: int(atomic.LoadInt32(&sb.unsealWait)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func addressToProverID(a address.Address) [32]byte {
|
|
||||||
var proverId [32]byte
|
|
||||||
copy(proverId[:], a.Payload())
|
|
||||||
return proverId
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) AcquireSectorId() (uint64, error) {
|
|
||||||
sb.idLk.Lock()
|
|
||||||
defer sb.idLk.Unlock()
|
|
||||||
|
|
||||||
sb.lastID++
|
|
||||||
id := sb.lastID
|
|
||||||
|
|
||||||
err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id)))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return id, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) AddPiece(pieceSize uint64, sectorId uint64, file io.Reader, existingPieceSizes []uint64) (PublicPieceInfo, error) {
|
|
||||||
fs := sb.filesystem
|
|
||||||
|
|
||||||
if err := fs.reserve(dataStaging, sb.ssize); err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
defer fs.free(dataStaging, sb.ssize)
|
|
||||||
|
|
||||||
atomic.AddInt32(&sb.addPieceWait, 1)
|
|
||||||
ret := sb.RateLimit()
|
|
||||||
atomic.AddInt32(&sb.addPieceWait, -1)
|
|
||||||
defer ret()
|
|
||||||
|
|
||||||
f, werr, err := toReadableFile(file, int64(pieceSize))
|
|
||||||
if err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
stagedFile, err := sb.stagedSectorFile(sectorId)
|
|
||||||
if err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _, commP, err := sectorbuilder.WriteWithAlignment(f, pieceSize, stagedFile, existingPieceSizes)
|
|
||||||
if err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := stagedFile.Close(); err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := f.Close(); err != nil {
|
|
||||||
return PublicPieceInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return PublicPieceInfo{
|
|
||||||
Size: pieceSize,
|
|
||||||
CommP: commP,
|
|
||||||
}, werr()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) ReadPieceFromSealedSector(sectorID uint64, offset uint64, size uint64, ticket []byte, commD []byte) (io.ReadCloser, error) {
|
|
||||||
fs := sb.filesystem
|
|
||||||
|
|
||||||
if err := fs.reserve(dataUnsealed, sb.ssize); err != nil { // TODO: this needs to get smarter when we start supporting partial unseals
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer fs.free(dataUnsealed, sb.ssize)
|
|
||||||
|
|
||||||
atomic.AddInt32(&sb.unsealWait, 1)
|
|
||||||
// TODO: Don't wait if cached
|
|
||||||
ret := sb.RateLimit() // TODO: check perf, consider remote unseal worker
|
|
||||||
defer ret()
|
|
||||||
atomic.AddInt32(&sb.unsealWait, -1)
|
|
||||||
|
|
||||||
sb.unsealLk.Lock() // TODO: allow unsealing unrelated sectors in parallel
|
|
||||||
defer sb.unsealLk.Unlock()
|
|
||||||
|
|
||||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
sealedPath, err := sb.SealedSectorPath(sectorID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
unsealedPath := sb.unsealedSectorPath(sectorID)
|
|
||||||
|
|
||||||
// TODO: GC for those
|
|
||||||
// (Probably configurable count of sectors to be kept unsealed, and just
|
|
||||||
// remove last used one (or use whatever other cache policy makes sense))
|
|
||||||
f, err := os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
|
|
||||||
if err != nil {
|
|
||||||
if !os.IsNotExist(err) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var commd [CommLen]byte
|
|
||||||
copy(commd[:], commD)
|
|
||||||
|
|
||||||
var tkt [CommLen]byte
|
|
||||||
copy(tkt[:], ticket)
|
|
||||||
|
|
||||||
err = sectorbuilder.Unseal(sb.ssize,
|
|
||||||
PoRepProofPartitions,
|
|
||||||
cacheDir,
|
|
||||||
sealedPath,
|
|
||||||
unsealedPath,
|
|
||||||
sectorID,
|
|
||||||
addressToProverID(sb.Miner),
|
|
||||||
tkt,
|
|
||||||
commd)
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("unseal failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err = os.OpenFile(unsealedPath, os.O_RDONLY, 0644)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
|
|
||||||
return nil, xerrors.Errorf("seek: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
lr := io.LimitReader(f, int64(size))
|
|
||||||
|
|
||||||
return &struct {
|
|
||||||
io.Reader
|
|
||||||
io.Closer
|
|
||||||
}{
|
|
||||||
Reader: lr,
|
|
||||||
Closer: f,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) sealPreCommitRemote(call workerCall) (RawSealPreCommitOutput, error) {
|
|
||||||
atomic.AddInt32(&sb.preCommitWait, -1)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ret := <-call.ret:
|
|
||||||
var err error
|
|
||||||
if ret.Err != "" {
|
|
||||||
err = xerrors.New(ret.Err)
|
|
||||||
}
|
|
||||||
return ret.Rspco.rspco(), err
|
|
||||||
case <-sb.stopping:
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.New("sectorbuilder stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) SealPreCommit(sectorID uint64, ticket SealTicket, pieces []PublicPieceInfo) (RawSealPreCommitOutput, error) {
|
|
||||||
fs := sb.filesystem
|
|
||||||
|
|
||||||
if err := fs.reserve(dataCache, sb.ssize); err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, err
|
|
||||||
}
|
|
||||||
defer fs.free(dataCache, sb.ssize)
|
|
||||||
|
|
||||||
if err := fs.reserve(dataSealed, sb.ssize); err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, err
|
|
||||||
}
|
|
||||||
defer fs.free(dataSealed, sb.ssize)
|
|
||||||
|
|
||||||
call := workerCall{
|
|
||||||
task: WorkerTask{
|
|
||||||
Type: WorkerPreCommit,
|
|
||||||
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
|
|
||||||
SectorID: sectorID,
|
|
||||||
SealTicket: ticket,
|
|
||||||
Pieces: pieces,
|
|
||||||
},
|
|
||||||
ret: make(chan SealRes),
|
|
||||||
}
|
|
||||||
|
|
||||||
atomic.AddInt32(&sb.preCommitWait, 1)
|
|
||||||
|
|
||||||
select { // prefer remote
|
|
||||||
case sb.precommitTasks <- call:
|
|
||||||
return sb.sealPreCommitRemote(call)
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.checkRateLimit()
|
|
||||||
|
|
||||||
rl := sb.rateLimit
|
|
||||||
if sb.noPreCommit {
|
|
||||||
rl = make(chan struct{})
|
|
||||||
}
|
|
||||||
|
|
||||||
select { // use whichever is available
|
|
||||||
case sb.precommitTasks <- call:
|
|
||||||
return sb.sealPreCommitRemote(call)
|
|
||||||
case rl <- struct{}{}:
|
|
||||||
}
|
|
||||||
|
|
||||||
atomic.AddInt32(&sb.preCommitWait, -1)
|
|
||||||
|
|
||||||
// local
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
<-sb.rateLimit
|
|
||||||
}()
|
|
||||||
|
|
||||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
|
||||||
if err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.Errorf("getting cache dir: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sealedPath, err := sb.SealedSectorPath(sectorID)
|
|
||||||
if err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.Errorf("getting sealed sector path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
e, err := os.OpenFile(sealedPath, os.O_RDWR|os.O_CREATE, 0644)
|
|
||||||
if err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.Errorf("ensuring sealed file exists: %w", err)
|
|
||||||
}
|
|
||||||
if err := e.Close(); err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var sum uint64
|
|
||||||
for _, piece := range pieces {
|
|
||||||
sum += piece.Size
|
|
||||||
}
|
|
||||||
ussize := UserBytesForSectorSize(sb.ssize)
|
|
||||||
if sum != ussize {
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
|
|
||||||
}
|
|
||||||
|
|
||||||
stagedPath := sb.StagedSectorPath(sectorID)
|
|
||||||
|
|
||||||
rspco, err := sectorbuilder.SealPreCommit(
|
|
||||||
sb.ssize,
|
|
||||||
PoRepProofPartitions,
|
|
||||||
cacheDir,
|
|
||||||
stagedPath,
|
|
||||||
sealedPath,
|
|
||||||
sectorID,
|
|
||||||
addressToProverID(sb.Miner),
|
|
||||||
ticket.TicketBytes,
|
|
||||||
pieces,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return RawSealPreCommitOutput{}, xerrors.Errorf("presealing sector %d (%s): %w", sectorID, stagedPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return RawSealPreCommitOutput(rspco), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) sealCommitRemote(call workerCall) (proof []byte, err error) {
|
|
||||||
atomic.AddInt32(&sb.commitWait, -1)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ret := <-call.ret:
|
|
||||||
if ret.Err != "" {
|
|
||||||
err = xerrors.New(ret.Err)
|
|
||||||
}
|
|
||||||
return ret.Proof, err
|
|
||||||
case <-sb.stopping:
|
|
||||||
return nil, xerrors.New("sectorbuilder stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) sealCommitLocal(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
|
|
||||||
atomic.AddInt32(&sb.commitWait, -1)
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
<-sb.rateLimit
|
|
||||||
}()
|
|
||||||
|
|
||||||
cacheDir, err := sb.sectorCacheDir(sectorID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
proof, err = sectorbuilder.SealCommit(
|
|
||||||
sb.ssize,
|
|
||||||
PoRepProofPartitions,
|
|
||||||
cacheDir,
|
|
||||||
sectorID,
|
|
||||||
addressToProverID(sb.Miner),
|
|
||||||
ticket.TicketBytes,
|
|
||||||
seed.TicketBytes,
|
|
||||||
pieces,
|
|
||||||
sectorbuilder.RawSealPreCommitOutput(rspco),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn("StandaloneSealCommit error: ", err)
|
|
||||||
log.Warnf("sid:%d tkt:%v seed:%v, ppi:%v rspco:%v", sectorID, ticket, seed, pieces, rspco)
|
|
||||||
|
|
||||||
return nil, xerrors.Errorf("StandaloneSealCommit: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return proof, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) SealCommit(sectorID uint64, ticket SealTicket, seed SealSeed, pieces []PublicPieceInfo, rspco RawSealPreCommitOutput) (proof []byte, err error) {
|
|
||||||
call := workerCall{
|
|
||||||
task: WorkerTask{
|
|
||||||
Type: WorkerCommit,
|
|
||||||
TaskID: atomic.AddUint64(&sb.taskCtr, 1),
|
|
||||||
SectorID: sectorID,
|
|
||||||
SealTicket: ticket,
|
|
||||||
Pieces: pieces,
|
|
||||||
|
|
||||||
SealSeed: seed,
|
|
||||||
Rspco: rspco,
|
|
||||||
},
|
|
||||||
ret: make(chan SealRes),
|
|
||||||
}
|
|
||||||
|
|
||||||
atomic.AddInt32(&sb.commitWait, 1)
|
|
||||||
|
|
||||||
select { // prefer remote
|
|
||||||
case sb.commitTasks <- call:
|
|
||||||
proof, err = sb.sealCommitRemote(call)
|
|
||||||
default:
|
|
||||||
sb.checkRateLimit()
|
|
||||||
|
|
||||||
rl := sb.rateLimit
|
|
||||||
if sb.noCommit {
|
|
||||||
rl = make(chan struct{})
|
|
||||||
}
|
|
||||||
|
|
||||||
select { // use whichever is available
|
|
||||||
case sb.commitTasks <- call:
|
|
||||||
proof, err = sb.sealCommitRemote(call)
|
|
||||||
case rl <- struct{}{}:
|
|
||||||
proof, err = sb.sealCommitLocal(sectorID, ticket, seed, pieces, rspco)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, xerrors.Errorf("commit: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return proof, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) ComputeElectionPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed []byte, winners []EPostCandidate) ([]byte, error) {
|
|
||||||
if len(challengeSeed) != CommLen {
|
|
||||||
return nil, xerrors.Errorf("given challenge seed was the wrong length: %d != %d", len(challengeSeed), CommLen)
|
|
||||||
}
|
|
||||||
var cseed [CommLen]byte
|
|
||||||
copy(cseed[:], challengeSeed)
|
|
||||||
|
|
||||||
privsects, err := sb.pubSectorToPriv(sectorInfo, nil) // TODO: faults
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
proverID := addressToProverID(sb.Miner)
|
|
||||||
|
|
||||||
return sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsects, cseed, winners)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, error) {
|
|
||||||
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
|
|
||||||
|
|
||||||
proverID := addressToProverID(sb.Miner)
|
|
||||||
return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) pubSectorToPriv(sectorInfo SortedPublicSectorInfo, faults []uint64) (SortedPrivateSectorInfo, error) {
|
|
||||||
fmap := map[uint64]struct{}{}
|
|
||||||
for _, fault := range faults {
|
|
||||||
fmap[fault] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
var out []sectorbuilder.PrivateSectorInfo
|
|
||||||
for _, s := range sectorInfo.Values() {
|
|
||||||
if _, faulty := fmap[s.SectorID]; faulty {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
cachePath, err := sb.sectorCacheDir(s.SectorID)
|
|
||||||
if err != nil {
|
|
||||||
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting cache path for sector %d: %w", s.SectorID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sealedPath, err := sb.SealedSectorPath(s.SectorID)
|
|
||||||
if err != nil {
|
|
||||||
return SortedPrivateSectorInfo{}, xerrors.Errorf("getting sealed path for sector %d: %w", s.SectorID, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out = append(out, sectorbuilder.PrivateSectorInfo{
|
|
||||||
SectorID: s.SectorID,
|
|
||||||
CommR: s.CommR,
|
|
||||||
CacheDirPath: cachePath,
|
|
||||||
SealedSectorPath: sealedPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return NewSortedPrivateSectorInfo(out), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) GenerateFallbackPoSt(sectorInfo SortedPublicSectorInfo, challengeSeed [CommLen]byte, faults []uint64) ([]EPostCandidate, []byte, error) {
|
|
||||||
privsectors, err := sb.pubSectorToPriv(sectorInfo, faults)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
challengeCount := fallbackPostChallengeCount(uint64(len(sectorInfo.Values())), len(faults))
|
|
||||||
|
|
||||||
proverID := addressToProverID(sb.Miner)
|
|
||||||
candidates, err := sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
proof, err := sectorbuilder.GeneratePoSt(sb.ssize, proverID, privsectors, challengeSeed, candidates)
|
|
||||||
return candidates, proof, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) Stop() {
|
|
||||||
close(sb.stopping)
|
|
||||||
}
|
|
||||||
|
|
||||||
func fallbackPostChallengeCount(sectors uint64, faults int) uint64 {
|
|
||||||
challengeCount := types.ElectionPostChallengeCount(sectors, faults)
|
|
||||||
if challengeCount > build.MaxFallbackPostChallengeCount {
|
|
||||||
return build.MaxFallbackPostChallengeCount
|
|
||||||
}
|
|
||||||
return challengeCount
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
|
|
||||||
if err := migrate(osb.filesystem.pathFor(dataCache), sb.filesystem.pathFor(dataCache), symlink); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := migrate(osb.filesystem.pathFor(dataStaging), sb.filesystem.pathFor(dataStaging), symlink); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := migrate(osb.filesystem.pathFor(dataSealed), sb.filesystem.pathFor(dataSealed), symlink); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
val, err := osb.ds.Get(lastSectorIdKey)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := sb.ds.Put(lastSectorIdKey, val); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.lastID = osb.lastID
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *SectorBuilder) SetLastSectorID(id uint64) error {
|
|
||||||
if err := sb.ds.Put(lastSectorIdKey, []byte(fmt.Sprint(id))); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.lastID = id
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func migrate(from, to string, symlink bool) error {
|
|
||||||
st, err := os.Stat(from)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if st.IsDir() {
|
|
||||||
return migrateDir(from, to, symlink)
|
|
||||||
}
|
|
||||||
return migrateFile(from, to, symlink)
|
|
||||||
}
|
|
||||||
|
|
||||||
func migrateDir(from, to string, symlink bool) error {
|
|
||||||
tost, err := os.Stat(to)
|
|
||||||
if err != nil {
|
|
||||||
if !os.IsNotExist(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := os.MkdirAll(to, 0755); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else if !tost.IsDir() {
|
|
||||||
return xerrors.Errorf("target %q already exists and is a file (expected directory)")
|
|
||||||
}
|
|
||||||
|
|
||||||
dirents, err := ioutil.ReadDir(from)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, inf := range dirents {
|
|
||||||
n := inf.Name()
|
|
||||||
if inf.IsDir() {
|
|
||||||
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := migrate(filepath.Join(from, n), filepath.Join(to, n), symlink); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func migrateFile(from, to string, symlink bool) error {
|
|
||||||
if symlink {
|
|
||||||
return os.Symlink(from, to)
|
|
||||||
}
|
|
||||||
|
|
||||||
return dcopy.Copy(from, to)
|
|
||||||
}
|
|
||||||
@ -1,365 +0,0 @@
|
|||||||
package sectorbuilder_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
ffi "github.com/filecoin-project/filecoin-ffi"
|
|
||||||
paramfetch "github.com/filecoin-project/go-paramfetch"
|
|
||||||
"github.com/ipfs/go-datastore"
|
|
||||||
logging "github.com/ipfs/go-log"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/lotus/build"
|
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
logging.SetLogLevel("*", "INFO")
|
|
||||||
}
|
|
||||||
|
|
||||||
const sectorSize = 1024
|
|
||||||
|
|
||||||
type seal struct {
|
|
||||||
sid uint64
|
|
||||||
|
|
||||||
pco sectorbuilder.RawSealPreCommitOutput
|
|
||||||
ppi sectorbuilder.PublicPieceInfo
|
|
||||||
|
|
||||||
ticket sectorbuilder.SealTicket
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *seal) precommit(t *testing.T, sb *sectorbuilder.SectorBuilder, sid uint64, done func()) {
|
|
||||||
dlen := sectorbuilder.UserBytesForSectorSize(sectorSize)
|
|
||||||
|
|
||||||
var err error
|
|
||||||
r := io.LimitReader(rand.New(rand.NewSource(42+int64(sid))), int64(dlen))
|
|
||||||
s.ppi, err = sb.AddPiece(dlen, sid, r, []uint64{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.ticket = sectorbuilder.SealTicket{
|
|
||||||
BlockHeight: 5,
|
|
||||||
TicketBytes: [32]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2},
|
|
||||||
}
|
|
||||||
|
|
||||||
s.pco, err = sb.SealPreCommit(sid, s.ticket, []sectorbuilder.PublicPieceInfo{s.ppi})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *seal) commit(t *testing.T, sb *sectorbuilder.SectorBuilder, done func()) {
|
|
||||||
seed := sectorbuilder.SealSeed{
|
|
||||||
BlockHeight: 15,
|
|
||||||
TicketBytes: [32]byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9},
|
|
||||||
}
|
|
||||||
|
|
||||||
proof, err := sb.SealCommit(s.sid, s.ticket, seed, []sectorbuilder.PublicPieceInfo{s.ppi}, s.pco)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ok, err := sectorbuilder.VerifySeal(sectorSize, s.pco.CommR[:], s.pco.CommD[:], sb.Miner, s.ticket.TicketBytes[:], seed.TicketBytes[:], s.sid, proof)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("proof failed to validate")
|
|
||||||
}
|
|
||||||
|
|
||||||
done()
|
|
||||||
}
|
|
||||||
|
|
||||||
func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Time {
|
|
||||||
cSeed := [32]byte{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9}
|
|
||||||
|
|
||||||
ppi := make([]ffi.PublicSectorInfo, len(seals))
|
|
||||||
for i, s := range seals {
|
|
||||||
ppi[i] = ffi.PublicSectorInfo{
|
|
||||||
SectorID: s.sid,
|
|
||||||
CommR: s.pco.CommR,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ssi := sectorbuilder.NewSortedPublicSectorInfo(ppi)
|
|
||||||
|
|
||||||
candndates, err := sb.GenerateEPostCandidates(ssi, cSeed, []uint64{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
genCandidates := time.Now()
|
|
||||||
|
|
||||||
if len(candndates) != 1 {
|
|
||||||
t.Fatal("expected 1 candidate")
|
|
||||||
}
|
|
||||||
|
|
||||||
postProof, err := sb.ComputeElectionPoSt(ssi, cSeed[:], candndates)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sb.SectorSize(), ssi, cSeed[:], postProof, candndates, sb.Miner)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("bad post")
|
|
||||||
}
|
|
||||||
|
|
||||||
return genCandidates
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSealAndVerify(t *testing.T) {
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("skipping test in short mode")
|
|
||||||
}
|
|
||||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
|
||||||
t.Skip("this is slow")
|
|
||||||
}
|
|
||||||
_ = os.Setenv("RUST_LOG", "info")
|
|
||||||
|
|
||||||
build.SectorSizes = []uint64{sectorSize}
|
|
||||||
|
|
||||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ds := datastore.NewMapDatastore()
|
|
||||||
|
|
||||||
dir, err := ioutil.TempDir("", "sbtest")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
cleanup := func() {
|
|
||||||
if t.Failed() {
|
|
||||||
fmt.Printf("not removing %s\n", dir)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := os.RemoveAll(dir); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
si, err := sb.AcquireSectorId()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := seal{sid: si}
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
s.precommit(t, sb, 1, func() {})
|
|
||||||
|
|
||||||
precommit := time.Now()
|
|
||||||
|
|
||||||
s.commit(t, sb, func() {})
|
|
||||||
|
|
||||||
commit := time.Now()
|
|
||||||
|
|
||||||
genCandidiates := post(t, sb, s)
|
|
||||||
|
|
||||||
epost := time.Now()
|
|
||||||
|
|
||||||
// Restart sectorbuilder, re-run post
|
|
||||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
post(t, sb, s)
|
|
||||||
|
|
||||||
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
|
|
||||||
fmt.Printf("Commit: %s\n", commit.Sub(precommit).String())
|
|
||||||
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String())
|
|
||||||
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSealPoStNoCommit(t *testing.T) {
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("skipping test in short mode")
|
|
||||||
}
|
|
||||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
|
||||||
t.Skip("this is slow")
|
|
||||||
}
|
|
||||||
_ = os.Setenv("RUST_LOG", "info")
|
|
||||||
|
|
||||||
build.SectorSizes = []uint64{sectorSize}
|
|
||||||
|
|
||||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ds := datastore.NewMapDatastore()
|
|
||||||
|
|
||||||
dir, err := ioutil.TempDir("", "sbtest")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
cleanup := func() {
|
|
||||||
if t.Failed() {
|
|
||||||
fmt.Printf("not removing %s\n", dir)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := os.RemoveAll(dir); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
si, err := sb.AcquireSectorId()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := seal{sid: si}
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
s.precommit(t, sb, 1, func() {})
|
|
||||||
|
|
||||||
precommit := time.Now()
|
|
||||||
|
|
||||||
// Restart sectorbuilder, re-run post
|
|
||||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := sb.TrimCache(1); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
genCandidiates := post(t, sb, s)
|
|
||||||
|
|
||||||
epost := time.Now()
|
|
||||||
|
|
||||||
fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String())
|
|
||||||
fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String())
|
|
||||||
fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSealAndVerify2(t *testing.T) {
|
|
||||||
if testing.Short() {
|
|
||||||
t.Skip("skipping test in short mode")
|
|
||||||
}
|
|
||||||
if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
|
|
||||||
t.Skip("this is slow")
|
|
||||||
}
|
|
||||||
_ = os.Setenv("RUST_LOG", "info")
|
|
||||||
|
|
||||||
build.SectorSizes = []uint64{sectorSize}
|
|
||||||
|
|
||||||
if err := paramfetch.GetParams(sectorSize); err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ds := datastore.NewMapDatastore()
|
|
||||||
|
|
||||||
dir, err := ioutil.TempDir("", "sbtest")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
cleanup := func() {
|
|
||||||
if err := os.RemoveAll(dir); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
defer cleanup()
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
|
|
||||||
si1, err := sb.AcquireSectorId()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
si2, err := sb.AcquireSectorId()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s1 := seal{sid: si1}
|
|
||||||
s2 := seal{sid: si2}
|
|
||||||
|
|
||||||
wg.Add(2)
|
|
||||||
go s1.precommit(t, sb, 1, wg.Done)
|
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
go s2.precommit(t, sb, 2, wg.Done)
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
wg.Add(2)
|
|
||||||
go s1.commit(t, sb, wg.Done)
|
|
||||||
go s2.commit(t, sb, wg.Done)
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
post(t, sb, s1, s2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAcquireID(t *testing.T) {
|
|
||||||
ds := datastore.NewMapDatastore()
|
|
||||||
|
|
||||||
dir, err := ioutil.TempDir("", "sbtest")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sb, err := sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertAcquire := func(expect uint64) {
|
|
||||||
id, err := sb.AcquireSectorId()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, expect, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertAcquire(1)
|
|
||||||
assertAcquire(2)
|
|
||||||
assertAcquire(3)
|
|
||||||
|
|
||||||
sb, err = sectorbuilder.TempSectorbuilderDir(dir, sectorSize, ds)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertAcquire(4)
|
|
||||||
assertAcquire(5)
|
|
||||||
assertAcquire(6)
|
|
||||||
|
|
||||||
if err := os.RemoveAll(dir); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,165 +0,0 @@
|
|||||||
package statestore
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"reflect"
|
|
||||||
|
|
||||||
"github.com/ipfs/go-datastore"
|
|
||||||
"github.com/ipfs/go-datastore/query"
|
|
||||||
cbg "github.com/whyrusleeping/cbor-gen"
|
|
||||||
"go.uber.org/multierr"
|
|
||||||
"golang.org/x/xerrors"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StateStore struct {
|
|
||||||
ds datastore.Datastore
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(ds datastore.Datastore) *StateStore {
|
|
||||||
return &StateStore{ds: ds}
|
|
||||||
}
|
|
||||||
|
|
||||||
func toKey(k interface{}) datastore.Key {
|
|
||||||
switch t := k.(type) {
|
|
||||||
case uint64:
|
|
||||||
return datastore.NewKey(fmt.Sprint(t))
|
|
||||||
case fmt.Stringer:
|
|
||||||
return datastore.NewKey(t.String())
|
|
||||||
default:
|
|
||||||
panic("unexpected key type")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *StateStore) Begin(i interface{}, state interface{}) error {
|
|
||||||
k := toKey(i)
|
|
||||||
has, err := st.ds.Has(k)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if has {
|
|
||||||
return xerrors.Errorf("already tracking state for %v", i)
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := cborutil.Dump(state)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.ds.Put(k, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *StateStore) End(i interface{}) error {
|
|
||||||
k := toKey(i)
|
|
||||||
has, err := st.ds.Has(k)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !has {
|
|
||||||
return xerrors.Errorf("No state for %s", i)
|
|
||||||
}
|
|
||||||
return st.ds.Delete(k)
|
|
||||||
}
|
|
||||||
|
|
||||||
func cborMutator(mutator interface{}) func([]byte) ([]byte, error) {
|
|
||||||
rmut := reflect.ValueOf(mutator)
|
|
||||||
|
|
||||||
return func(in []byte) ([]byte, error) {
|
|
||||||
state := reflect.New(rmut.Type().In(0).Elem())
|
|
||||||
|
|
||||||
err := cborutil.ReadCborRPC(bytes.NewReader(in), state.Interface())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
out := rmut.Call([]reflect.Value{state})
|
|
||||||
|
|
||||||
if err := out[0].Interface(); err != nil {
|
|
||||||
return nil, err.(error)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cborutil.Dump(state.Interface())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mutator func(*T) error
|
|
||||||
func (st *StateStore) Mutate(i interface{}, mutator interface{}) error {
|
|
||||||
return st.mutate(i, cborMutator(mutator))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *StateStore) mutate(i interface{}, mutator func([]byte) ([]byte, error)) error {
|
|
||||||
k := toKey(i)
|
|
||||||
has, err := st.ds.Has(k)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !has {
|
|
||||||
return xerrors.Errorf("No state for %s", i)
|
|
||||||
}
|
|
||||||
|
|
||||||
cur, err := st.ds.Get(k)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
mutated, err := mutator(cur)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.ds.Put(k, mutated)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *StateStore) Has(i interface{}) (bool, error) {
|
|
||||||
return st.ds.Has(toKey(i))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *StateStore) Get(i interface{}, out cbg.CBORUnmarshaler) error {
|
|
||||||
k := toKey(i)
|
|
||||||
val, err := st.ds.Get(k)
|
|
||||||
if err != nil {
|
|
||||||
if xerrors.Is(err, datastore.ErrNotFound) {
|
|
||||||
return xerrors.Errorf("No state for %s: %w", i, err)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return out.UnmarshalCBOR(bytes.NewReader(val))
|
|
||||||
}
|
|
||||||
|
|
||||||
// out: *[]T
|
|
||||||
func (st *StateStore) List(out interface{}) error {
|
|
||||||
res, err := st.ds.Query(query.Query{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer res.Close()
|
|
||||||
|
|
||||||
outT := reflect.TypeOf(out).Elem().Elem()
|
|
||||||
rout := reflect.ValueOf(out)
|
|
||||||
|
|
||||||
var errs error
|
|
||||||
|
|
||||||
for {
|
|
||||||
res, ok := res.NextSync()
|
|
||||||
if !ok {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if res.Error != nil {
|
|
||||||
return res.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
elem := reflect.New(outT)
|
|
||||||
err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
|
|
||||||
if err != nil {
|
|
||||||
errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -1,38 +0,0 @@
|
|||||||
package statestore
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/ipfs/go-datastore"
|
|
||||||
|
|
||||||
"github.com/filecoin-project/go-cbor-util"
|
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestList(t *testing.T) {
|
|
||||||
ds := datastore.NewMapDatastore()
|
|
||||||
|
|
||||||
e, err := cborutil.Dump(types.NewInt(7))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := ds.Put(datastore.NewKey("/2"), e); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
st := &StateStore{ds: ds}
|
|
||||||
|
|
||||||
var out []types.BigInt
|
|
||||||
if err := st.List(&out); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(out) != 1 {
|
|
||||||
t.Fatal("wrong len")
|
|
||||||
}
|
|
||||||
|
|
||||||
if out[0].Int64() != 7 {
|
|
||||||
t.Fatal("wrong data")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -13,8 +13,8 @@ import (
|
|||||||
files "github.com/ipfs/go-ipfs-files"
|
files "github.com/ipfs/go-ipfs-files"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
"github.com/filecoin-project/lotus/lib/tarutil"
|
"github.com/filecoin-project/lotus/lib/tarutil"
|
||||||
"github.com/filecoin-project/lotus/miner"
|
"github.com/filecoin-project/lotus/miner"
|
||||||
"github.com/filecoin-project/lotus/storage"
|
"github.com/filecoin-project/lotus/storage"
|
||||||
|
|||||||
@ -20,13 +20,13 @@ import (
|
|||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/deals"
|
"github.com/filecoin-project/lotus/chain/deals"
|
||||||
"github.com/filecoin-project/lotus/chain/gen"
|
"github.com/filecoin-project/lotus/chain/gen"
|
||||||
"github.com/filecoin-project/lotus/datatransfer"
|
"github.com/filecoin-project/lotus/datatransfer"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
"github.com/filecoin-project/lotus/miner"
|
"github.com/filecoin-project/lotus/miner"
|
||||||
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
||||||
"github.com/filecoin-project/lotus/node/modules/helpers"
|
"github.com/filecoin-project/lotus/node/modules/helpers"
|
||||||
|
|||||||
@ -13,14 +13,14 @@ import (
|
|||||||
"golang.org/x/xerrors"
|
"golang.org/x/xerrors"
|
||||||
|
|
||||||
"github.com/filecoin-project/go-address"
|
"github.com/filecoin-project/go-address"
|
||||||
|
"github.com/filecoin-project/go-sectorbuilder"
|
||||||
|
"github.com/filecoin-project/go-statestore"
|
||||||
"github.com/filecoin-project/lotus/api"
|
"github.com/filecoin-project/lotus/api"
|
||||||
"github.com/filecoin-project/lotus/build"
|
"github.com/filecoin-project/lotus/build"
|
||||||
"github.com/filecoin-project/lotus/chain/events"
|
"github.com/filecoin-project/lotus/chain/events"
|
||||||
"github.com/filecoin-project/lotus/chain/gen"
|
"github.com/filecoin-project/lotus/chain/gen"
|
||||||
"github.com/filecoin-project/lotus/chain/store"
|
"github.com/filecoin-project/lotus/chain/store"
|
||||||
"github.com/filecoin-project/lotus/chain/types"
|
"github.com/filecoin-project/lotus/chain/types"
|
||||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
|
||||||
"github.com/filecoin-project/lotus/lib/statestore"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var log = logging.Logger("storageminer")
|
var log = logging.Logger("storageminer")
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user