diff --git a/api/api_full.go b/api/api_full.go index 1a3850cc6..184805698 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -122,10 +122,10 @@ type FullNode interface { // It fails if message fails to execute. GasEstimateGasLimit(context.Context, *types.Message, types.TipSetKey) (int64, error) - // GasEsitmateGasPremium estimates what gas price should be used for a + // GasEstimateGasPremium estimates what gas price should be used for a // message to have high likelihood of inclusion in `nblocksincl` epochs. - GasEsitmateGasPremium(_ context.Context, nblocksincl uint64, + GasEstimateGasPremium(_ context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) // MethodGroup: Sync diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index bc3b72a17..a571e4564 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -87,7 +87,7 @@ type FullNodeStruct struct { BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"` - GasEsitmateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"` + GasEstimateGasPremium func(context.Context, uint64, address.Address, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"` GasEstimateGasLimit func(context.Context, *types.Message, types.TipSetKey) (int64, error) `perm:"read"` GasEstimateFeeCap func(context.Context, *types.Message, int64, types.TipSetKey) (types.BigInt, error) `perm:"read"` @@ -257,7 +257,7 @@ type StorageMinerStruct struct { StorageAttach func(context.Context, stores.StorageInfo, fsutil.FsStat) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType, bool) error `perm:"admin"` StorageDropSector func(context.Context, stores.ID, abi.SectorID, stores.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, stores.SectorFileType, abi.RegisteredSealProof, bool) ([]stores.SectorStorageInfo, error) `perm:"admin"` StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate stores.SectorFileType, spt abi.RegisteredSealProof, sealing stores.PathType) ([]stores.StorageInfo, error) `perm:"admin"` StorageReportHealth func(ctx context.Context, id stores.ID, report stores.HealthReport) error `perm:"admin"` @@ -434,9 +434,9 @@ func (c *FullNodeStruct) ClientDealSize(ctx context.Context, root cid.Cid) (api. return c.Internal.ClientDealSize(ctx, root) } -func (c *FullNodeStruct) GasEsitmateGasPremium(ctx context.Context, nblocksincl uint64, +func (c *FullNodeStruct) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) { - return c.Internal.GasEsitmateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk) + return c.Internal.GasEstimateGasPremium(ctx, nblocksincl, sender, gaslimit, tsk) } func (c *FullNodeStruct) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64, tsk types.TipSetKey) (types.BigInt, error) { @@ -980,8 +980,8 @@ func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId st return c.Internal.StorageDropSector(ctx, storageId, s, ft) } -func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, allowFetch bool) ([]stores.SectorStorageInfo, error) { - return c.Internal.StorageFindSector(ctx, si, types, allowFetch) +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types stores.SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]stores.SectorStorageInfo, error) { + return c.Internal.StorageFindSector(ctx, si, types, spt, allowFetch) } func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) { diff --git a/api/test/window_post.go b/api/test/window_post.go index a31d17825..4ff02afa4 100644 --- a/api/test/window_post.go +++ b/api/test/window_post.go @@ -44,16 +44,11 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect mine := true done := make(chan struct{}) - blockNotif := make(chan struct{}, 1) go func() { defer close(done) for mine { build.Clock.Sleep(blocktime) if err := sn[0].MineOne(ctx, bminer.MineReq{Done: func(bool, error) { - select { - case blockNotif <- struct{}{}: - default: - } }}); err != nil { t.Error(err) @@ -61,7 +56,7 @@ func TestPledgeSector(t *testing.T, b APIBuilder, blocktime time.Duration, nSect } }() - pledgeSectors(t, ctx, miner, nSectors, 0, blockNotif) + pledgeSectors(t, ctx, miner, nSectors, 0, nil) mine = false <-done diff --git a/build/version.go b/build/version.go index b7fe7fa94..21cec2494 100644 --- a/build/version.go +++ b/build/version.go @@ -25,7 +25,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.4.3" +const BuildVersion = "0.4.4" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -53,7 +53,7 @@ func (ve Version) EqMajorMinor(v2 Version) bool { } // APIVersion is a semver version of the rpc api exposed -var APIVersion Version = newVer(0, 9, 0) +var APIVersion Version = newVer(0, 10, 0) //nolint:varcheck,deadcode const ( diff --git a/chain/blocksync/client.go b/chain/blocksync/client.go index 87cb41d44..bc1826527 100644 --- a/chain/blocksync/client.go +++ b/chain/blocksync/client.go @@ -393,7 +393,7 @@ func (client *BlockSync) sendRequestToPeer( &res) if err != nil { client.peerTracker.logFailure(peer, build.Clock.Since(connectionStart)) - return nil, err + return nil, xerrors.Errorf("failed to read blocksync response: %w", err) } // FIXME: Move all this together at the top using a defer as done elsewhere. diff --git a/chain/messagepool/config.go b/chain/messagepool/config.go index 4e1b07062..eaf6ca75b 100644 --- a/chain/messagepool/config.go +++ b/chain/messagepool/config.go @@ -14,6 +14,7 @@ var ( MemPoolSizeLimitHiDefault = 30000 MemPoolSizeLimitLoDefault = 20000 PruneCooldownDefault = time.Minute + GasLimitOverestimation = 1.25 ConfigKey = datastore.NewKey("/mpool/config") ) @@ -34,6 +35,10 @@ func loadConfig(ds dtypes.MetadataDS) (*types.MpoolConfig, error) { } cfg := new(types.MpoolConfig) err = json.Unmarshal(cfgBytes, cfg) + if cfg.GasLimitOverestimation == 0 { + // TODO: remove in next reset + cfg.GasLimitOverestimation = GasLimitOverestimation + } return cfg, err } @@ -65,9 +70,10 @@ func (mp *MessagePool) SetConfig(cfg *types.MpoolConfig) { func DefaultConfig() *types.MpoolConfig { return &types.MpoolConfig{ - SizeLimitHigh: MemPoolSizeLimitHiDefault, - SizeLimitLow: MemPoolSizeLimitLoDefault, - ReplaceByFeeRatio: ReplaceByFeeRatioDefault, - PruneCooldown: PruneCooldownDefault, + SizeLimitHigh: MemPoolSizeLimitHiDefault, + SizeLimitLow: MemPoolSizeLimitLoDefault, + ReplaceByFeeRatio: ReplaceByFeeRatioDefault, + PruneCooldown: PruneCooldownDefault, + GasLimitOverestimation: GasLimitOverestimation, } } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 799cb79b5..0d47e488e 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -55,7 +55,8 @@ var ( ErrInvalidToAddr = errors.New("message had invalid to address") - ErrBroadcastAnyway = errors.New("broadcasting message despite validation fail") + ErrBroadcastAnyway = errors.New("broadcasting message despite validation fail") + ErrRBFTooLowPremium = errors.New("replace by fee has too low GasPremium") ) const ( @@ -135,8 +136,9 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool) (bool, error) { } else { log.Info("add with duplicate nonce") return false, xerrors.Errorf("message from %s with nonce %d already in mpool,"+ - " increase GasPremium to %s from %s to trigger replace by fee", - m.Message.From, m.Message.Nonce, minPrice, m.Message.GasPremium) + " increase GasPremium to %s from %s to trigger replace by fee: %w", + m.Message.From, m.Message.Nonce, minPrice, m.Message.GasPremium, + ErrRBFTooLowPremium) } } } @@ -517,7 +519,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage) error { incr, err := mset.add(m, mp) if err != nil { log.Info(err) - return err // TODO(review): this error return was dropped at some point, was it on purpose? + return err } if incr { diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index 9c4b0d2e1..47923fd6f 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -59,7 +59,7 @@ func (mp *MessagePool) selectMessages(curTs, ts *types.TipSet) ([]*types.SignedM // defer only here so if we have no pending messages we don't spam defer func() { - log.Infof("message selection took %s", time.Since(start)) + log.Infow("message selection done", "took", time.Since(start)) }() // 0b. Select all priority messages that fit in the block @@ -72,11 +72,13 @@ func (mp *MessagePool) selectMessages(curTs, ts *types.TipSet) ([]*types.SignedM } // 1. Create a list of dependent message chains with maximal gas reward per limit consumed + startChains := time.Now() var chains []*msgChain for actor, mset := range pending { next := mp.createMessageChains(actor, mset, baseFee, ts) chains = append(chains, next...) } + log.Infow("create message chains done", "took", time.Since(startChains)) // 2. Sort the chains sort.Slice(chains, func(i, j int) bool { @@ -90,6 +92,7 @@ func (mp *MessagePool) selectMessages(curTs, ts *types.TipSet) ([]*types.SignedM // 3. Merge the head chains to produce the list of messages selected for inclusion, subject to // the block gas limit. + startMerge := time.Now() last := len(chains) for i, chain := range chains { // does it fit in the block? @@ -108,6 +111,7 @@ func (mp *MessagePool) selectMessages(curTs, ts *types.TipSet) ([]*types.SignedM last = i break } + log.Infow("merge message chains done", "took", time.Since(startMerge)) // 4. We have reached the edge of what we can fit wholesale; if we still have available gasLimit // to pack some more chains, then trim the last chain and push it down. @@ -115,6 +119,7 @@ func (mp *MessagePool) selectMessages(curTs, ts *types.TipSet) ([]*types.SignedM // dependency cannot be (fully) included. // We do this in a loop because the blocker might have been inordinately large and we might // have to do it multiple times to satisfy tail packing. + startTail := time.Now() tailLoop: for gasLimit >= minGas && last < len(chains) { // trim @@ -157,11 +162,17 @@ tailLoop: // -- mark the end. last = len(chains) } + log.Infow("pack tail chains done", "took", time.Since(startTail)) return result, nil } func (mp *MessagePool) selectPriorityMessages(pending map[address.Address]map[uint64]*types.SignedMessage, baseFee types.BigInt, ts *types.TipSet) ([]*types.SignedMessage, int64) { + start := time.Now() + defer func() { + log.Infow("select priority messages done", "took", time.Since(start)) + }() + result := make([]*types.SignedMessage, 0, mp.cfg.SizeLimitLow) gasLimit := int64(build.BlockGasLimit) minGas := int64(gasguess.MinGas) @@ -242,8 +253,15 @@ tailLoop: } func (mp *MessagePool) getPendingMessages(curTs, ts *types.TipSet) (map[address.Address]map[uint64]*types.SignedMessage, error) { + start := time.Now() + result := make(map[address.Address]map[uint64]*types.SignedMessage) haveCids := make(map[cid.Cid]struct{}) + defer func() { + if time.Since(start) > time.Millisecond { + log.Infow("get pending messages done", "took", time.Since(start)) + } + }() // are we in sync? inSync := false diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index 7fbc4b1b1..177c62dc2 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -286,6 +286,78 @@ func TestMessageChains(t *testing.T) { } +func TestMessageChainSkipping(t *testing.T) { + // regression test for chain skip bug + + mp, tma := makeTestMpool() + + // the actors + w1, err := wallet.NewWallet(wallet.NewMemKeyStore()) + if err != nil { + t.Fatal(err) + } + + a1, err := w1.GenerateKey(crypto.SigTypeBLS) + if err != nil { + t.Fatal(err) + } + + w2, err := wallet.NewWallet(wallet.NewMemKeyStore()) + if err != nil { + t.Fatal(err) + } + + a2, err := w2.GenerateKey(crypto.SigTypeBLS) + if err != nil { + t.Fatal(err) + } + + block := mock.MkBlock(nil, 1, 1) + ts := mock.TipSet(block) + + gasLimit := gasguess.Costs[gasguess.CostKey{builtin.StorageMarketActorCodeID, 2}] + baseFee := types.NewInt(0) + + tma.setBalance(a1, 1) // in FIL + tma.setStateNonce(a1, 10) + + mset := make(map[uint64]*types.SignedMessage) + for i := 0; i < 20; i++ { + bias := (20 - i) / 3 + m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(1+i%3+bias)) + mset[uint64(i)] = m + } + + chains := mp.createMessageChains(a1, mset, baseFee, ts) + if len(chains) != 4 { + t.Fatalf("expected 4 chains, got %d", len(chains)) + } + for i, chain := range chains { + var expectedLen int + switch { + case i == 0: + expectedLen = 2 + case i > 2: + expectedLen = 2 + default: + expectedLen = 3 + } + if len(chain.msgs) != expectedLen { + t.Fatalf("expected %d message in chain %d but got %d", expectedLen, i, len(chain.msgs)) + } + } + nextNonce := 10 + for _, chain := range chains { + for _, m := range chain.msgs { + if m.Message.Nonce != uint64(nextNonce) { + t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce) + } + nextNonce++ + } + } + +} + func TestBasicMessageSelection(t *testing.T) { mp, tma := makeTestMpool() @@ -492,3 +564,82 @@ func TestMessageSelectionTrimming(t *testing.T) { } } + +func TestPriorityMessageSelection(t *testing.T) { + mp, tma := makeTestMpool() + + // the actors + w1, err := wallet.NewWallet(wallet.NewMemKeyStore()) + if err != nil { + t.Fatal(err) + } + + a1, err := w1.GenerateKey(crypto.SigTypeBLS) + if err != nil { + t.Fatal(err) + } + + w2, err := wallet.NewWallet(wallet.NewMemKeyStore()) + if err != nil { + t.Fatal(err) + } + + a2, err := w2.GenerateKey(crypto.SigTypeBLS) + if err != nil { + t.Fatal(err) + } + + block := mock.MkBlock(nil, 1, 1) + ts := mock.TipSet(block) + tma.applyBlock(t, block) + + gasLimit := gasguess.Costs[gasguess.CostKey{builtin.StorageMarketActorCodeID, 2}] + + tma.setBalance(a1, 1) // in FIL + tma.setBalance(a2, 1) // in FIL + + mp.cfg.PriorityAddrs = []address.Address{a1} + + nMessages := 10 + for i := 0; i < nMessages; i++ { + bias := (nMessages - i) / 3 + m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(1+i%3+bias)) + mustAdd(t, mp, m) + m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(1+i%3+bias)) + mustAdd(t, mp, m) + } + + msgs, err := mp.SelectMessages(ts) + if err != nil { + t.Fatal(err) + } + + if len(msgs) != 20 { + t.Fatalf("expected 20 messages but got %d", len(msgs)) + } + + // messages from a1 must be first + nextNonce := uint64(0) + for i := 0; i < 10; i++ { + m := msgs[i] + if m.Message.From != a1 { + t.Fatal("expected messages from a1 before messages from a2") + } + if m.Message.Nonce != nextNonce { + t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce) + } + nextNonce++ + } + + nextNonce = 0 + for i := 10; i < 20; i++ { + m := msgs[i] + if m.Message.From != a2 { + t.Fatal("expected messages from a2 after messages from a1") + } + if m.Message.Nonce != nextNonce { + t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce) + } + nextNonce++ + } +} diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index 286d1dae7..19e241fce 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -108,7 +108,10 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri ts = sm.cs.GetHeaviestTipSet() } - state := ts.ParentState() + state, _, err := sm.TipSetState(ctx, ts) + if err != nil { + return nil, xerrors.Errorf("computing tipset state: %w", err) + } r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height()) @@ -122,7 +125,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri vmopt := &vm.VMOpts{ StateBase: state, - Epoch: ts.Height(), + Epoch: ts.Height() + 1, Rand: r, Bstore: sm.cs.Blockstore(), Syscalls: sm.cs.VMSys(), diff --git a/chain/sub/incoming.go b/chain/sub/incoming.go index 86f8eeec9..75f23cd3b 100644 --- a/chain/sub/incoming.go +++ b/chain/sub/incoming.go @@ -510,7 +510,7 @@ func (mv *MessageValidator) Validate(ctx context.Context, pid peer.ID, msg *pubs ) stats.Record(ctx, metrics.MessageValidationFailure.M(1)) switch { - case xerrors.Is(err, messagepool.ErrBroadcastAnyway): + case xerrors.Is(err, messagepool.ErrBroadcastAnyway) || xerrors.Is(err, messagepool.ErrRBFTooLowPremium): return pubsub.ValidationIgnore default: return pubsub.ValidationReject diff --git a/chain/types/mpool.go b/chain/types/mpool.go index cd6a99a47..cf08177e9 100644 --- a/chain/types/mpool.go +++ b/chain/types/mpool.go @@ -7,11 +7,12 @@ import ( ) type MpoolConfig struct { - PriorityAddrs []address.Address - SizeLimitHigh int - SizeLimitLow int - ReplaceByFeeRatio float64 - PruneCooldown time.Duration + PriorityAddrs []address.Address + SizeLimitHigh int + SizeLimitLow int + ReplaceByFeeRatio float64 + PruneCooldown time.Duration + GasLimitOverestimation float64 } func (mc *MpoolConfig) Clone() *MpoolConfig { diff --git a/chain/vm/burn.go b/chain/vm/burn.go index bad132de9..e9b6802c1 100644 --- a/chain/vm/burn.go +++ b/chain/vm/burn.go @@ -29,12 +29,12 @@ func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { return 0, gasLimit } - // over = gasLimit/gasUsed - 1 - 0.3 + // over = gasLimit/gasUsed - 1 - 0.1 // over = min(over, 1) // gasToBurn = (gasLimit - gasUsed) * over // so to factor out division from `over` - // over*gasUsed = min(gasLimit - (13*gasUsed)/10, gasUsed) + // over*gasUsed = min(gasLimit - (11*gasUsed)/10, gasUsed) // gasToBurn = ((gasLimit - gasUsed)*over*gasUsed) / gasUsed over := gasLimit - (gasOveruseNum*gasUsed)/gasOveruseDenom if over < 0 { diff --git a/cli/chain.go b/cli/chain.go index 12f632b36..003b3ccfc 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -419,7 +419,8 @@ var chainListCmd = &cli.Command{ } tss = otss for i, ts := range tss { - fmt.Printf("%d: %d blocks\n", ts.Height(), len(ts.Blocks())) + pbf := ts.Blocks()[0].ParentBaseFee + fmt.Printf("%d: %d blocks (baseFee: %s -> maxFee: %s)\n", ts.Height(), len(ts.Blocks()), ts.Blocks()[0].ParentBaseFee, types.FIL(types.BigMul(pbf, types.NewInt(uint64(build.BlockGasLimit))))) for _, b := range ts.Blocks() { msgs, err := api.ChainGetBlockMessages(ctx, b.Cid()) @@ -445,7 +446,7 @@ var chainListCmd = &cli.Command{ avgpremium = big.Div(psum, big.NewInt(int64(lenmsgs))) } - fmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPrice: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium) + fmt.Printf("\t%s: \t%d msgs, gasLimit: %d / %d (%0.2f%%), avgPremium: %s\n", b.Miner, len(msgs.BlsMessages)+len(msgs.SecpkMessages), limitSum, build.BlockGasLimit, 100*float64(limitSum)/float64(build.BlockGasLimit), avgpremium) } if i < len(tss)-1 { msgs, err := api.ChainGetParentMessages(ctx, tss[i+1].Blocks()[0].Cid()) @@ -1030,9 +1031,9 @@ var chainGasPriceCmd = &cli.Command{ nb := []int{1, 2, 3, 5, 10, 20, 50, 100, 300} for _, nblocks := range nb { - addr := builtin.SystemActorAddr // TODO: make real when used in GasEsitmateGasPremium + addr := builtin.SystemActorAddr // TODO: make real when used in GasEstimateGasPremium - est, err := api.GasEsitmateGasPremium(ctx, uint64(nblocks), addr, 10000, types.EmptyTSK) + est, err := api.GasEstimateGasPremium(ctx, uint64(nblocks), addr, 10000, types.EmptyTSK) if err != nil { return err } diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index aed67408a..43b0b2ffc 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -205,7 +205,7 @@ type processTipSetApi interface { StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) - GasEsitmateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) + GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) WalletBalance(ctx context.Context, addr address.Address) (types.BigInt, error) } @@ -293,7 +293,7 @@ func ProcessTipset(ctx context.Context, api processTipSetApi, tipset *types.TipS refundValue := refunds.GetRefund(maddr) // We want to try and ensure these messages get mined quickly - gasPremium, err := api.GasEsitmateGasPremium(ctx, 0, wallet, 0, tipset.Key()) + gasPremium, err := api.GasEstimateGasPremium(ctx, 0, wallet, 0, tipset.Key()) if err != nil { log.Warnw("failed to estimate gas premium", "err", err) continue diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 26a36431e..712962c9b 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -294,17 +294,17 @@ var storageFindCmd = &cli.Command{ Number: abi.SectorNumber(snum), } - u, err := nodeApi.StorageFindSector(ctx, sid, stores.FTUnsealed, false) + u, err := nodeApi.StorageFindSector(ctx, sid, stores.FTUnsealed, 0, false) if err != nil { return xerrors.Errorf("finding unsealed: %w", err) } - s, err := nodeApi.StorageFindSector(ctx, sid, stores.FTSealed, false) + s, err := nodeApi.StorageFindSector(ctx, sid, stores.FTSealed, 0, false) if err != nil { return xerrors.Errorf("finding sealed: %w", err) } - c, err := nodeApi.StorageFindSector(ctx, sid, stores.FTCache, false) + c, err := nodeApi.StorageFindSector(ctx, sid, stores.FTCache, 0, false) if err != nil { return xerrors.Errorf("finding cache: %w", err) } diff --git a/extern/sector-storage/.circleci/config.yml b/extern/sector-storage/.circleci/config.yml new file mode 100644 index 000000000..339fd4d4d --- /dev/null +++ b/extern/sector-storage/.circleci/config.yml @@ -0,0 +1,79 @@ +version: 2.1 +orbs: + go: gotest/tools@0.0.9 +executors: + golang: + docker: + - image: circleci/golang:1.13 + resource_class: 2xlarge +commands: + prepare-git-checkout: + steps: + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive + install-build-dependencies: + steps: + - run: sudo apt-get update + - run: sudo apt-get install -y jq ocl-icd-opencl-dev + - run: ./extern/filecoin-ffi/install-filcrypto + download-groth-params-and-verifying-keys: + steps: + - restore_cache: + name: Restore parameters cache + keys: + - 'v26a-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + - run: | + DIR=$(pwd) + cd $(mktemp -d) + go get github.com/filecoin-project/go-paramfetch/paramfetch + go build -o go-paramfetch github.com/filecoin-project/go-paramfetch/paramfetch + ./go-paramfetch 2048 "${DIR}/parameters.json" + - save_cache: + name: Save parameters cache + key: 'v26a-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ +jobs: + test: + executor: golang + environment: + RUST_LOG: info + steps: + - prepare-git-checkout + - install-build-dependencies + - download-groth-params-and-verifying-keys + - run: go test -v -timeout 10m ./... + mod-tidy-check: + executor: golang + steps: + - prepare-git-checkout + - go/mod-download + - go/mod-tidy-check + gofmt-check: + executor: golang + steps: + - prepare-git-checkout + - go/mod-download + - run: "! go fmt ./... 2>&1 | read" + lint-check: + executor: golang + steps: + - prepare-git-checkout + - install-build-dependencies + - go/mod-download + - go/install-golangci-lint: + gobin: $HOME/.local/bin + version: 1.23.8 + - run: + command: $HOME/.local/bin/golangci-lint run -v --concurrency 2 +workflows: + version: 2.1 + build_and_test: + jobs: + - mod-tidy-check + - lint-check + - gofmt-check + - test diff --git a/extern/sector-storage/.gitignore b/extern/sector-storage/.gitignore new file mode 100644 index 000000000..c90dde94c --- /dev/null +++ b/extern/sector-storage/.gitignore @@ -0,0 +1,2 @@ +.update-modules +.filecoin-build diff --git a/extern/sector-storage/.gitmodules b/extern/sector-storage/.gitmodules new file mode 100644 index 000000000..a655f05b9 --- /dev/null +++ b/extern/sector-storage/.gitmodules @@ -0,0 +1,4 @@ +[submodule "extern/filecoin-ffi"] + path = extern/filecoin-ffi + url = https://github.com/filecoin-project/filecoin-ffi.git + branch = master diff --git a/extern/sector-storage/LICENSE-APACHE b/extern/sector-storage/LICENSE-APACHE new file mode 100644 index 000000000..14478a3b6 --- /dev/null +++ b/extern/sector-storage/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/extern/sector-storage/LICENSE-MIT b/extern/sector-storage/LICENSE-MIT new file mode 100644 index 000000000..72dc60d84 --- /dev/null +++ b/extern/sector-storage/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/extern/sector-storage/Makefile b/extern/sector-storage/Makefile new file mode 100644 index 000000000..7b8d49683 --- /dev/null +++ b/extern/sector-storage/Makefile @@ -0,0 +1,29 @@ +all: build +.PHONY: all + +SUBMODULES= + +FFI_PATH:=./extern/filecoin-ffi/ +FFI_DEPS:=.install-filcrypto +FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS)) + +$(FFI_DEPS): .filecoin-build ; + +.filecoin-build: $(FFI_PATH) + $(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%) + @touch $@ + +.update-modules: + git submodule update --init --recursive + @touch $@ + +test: .update-modules .filecoin-build + go test -v ./... +.PHONY: test +SUBMODULES+=test + +build: $(SUBMODULES) + +clean: + rm -f .filecoin-build + rm -f .update-modules diff --git a/extern/sector-storage/README.md b/extern/sector-storage/README.md new file mode 100644 index 000000000..a4661f9d8 --- /dev/null +++ b/extern/sector-storage/README.md @@ -0,0 +1,61 @@ +# sector-storage + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![CircleCI](https://circleci.com/gh/filecoin-project/sector-storage.svg?style=svg)](https://circleci.com/gh/filecoin-project/sector-storage) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) + +> a concrete implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface + +The sector-storage project provides a implementation-nonspecific reference implementation of the [specs-storage](https://github.com/filecoin-project/specs-storage) interface. + +## Disclaimer + +Please report your issues with regards to sector-storage at the [lotus issue tracker](https://github.com/filecoin-project/lotus/issues) + +## Architecture + +![high-level architecture](docs/sector-storage.svg) + +### `Manager` + +Manages is the top-level piece of the storage system gluing all the other pieces +together. It also implements scheduling logic. + +### `package stores` + +This package implements the sector storage subsystem. Fundamentally the storage +is divided into `path`s, each path has it's UUID, and stores a set of sector +'files'. There are currently 3 types of sector files - `unsealed`, `sealed`, +and `cache`. + +Paths can be shared between nodes by sharing the underlying filesystem. + +### `stores.Local` + +The Local store implements SectorProvider for paths mounted in the local +filesystem. Paths can be shared between nodes, and support shared filesystems +such as NFS. + +stores.Local implements all native filesystem-related operations + +### `stores.Remote` + +The Remote store extends Local store, handles fetching sector files into a local +store if needed, and handles removing sectors from non-local stores. + +### `stores.Index` + +The Index is a singleton holding metadata about storage paths, and a mapping of +sector files to paths + +### `LocalWorker` + +LocalWorker implements the Worker interface with ffiwrapper.Sealer and a +store.Store instance + +## License + +The Filecoin Project is dual-licensed under Apache 2.0 and MIT terms: + +- Apache License, Version 2.0, ([LICENSE-APACHE](https://github.com/filecoin-project/sector-storage/blob/master/LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](https://github.com/filecoin-project/sector-storage/blob/master/LICENSE-MIT) or http://opensource.org/licenses/MIT) diff --git a/extern/sector-storage/docs/sector-storage.svg b/extern/sector-storage/docs/sector-storage.svg new file mode 100644 index 000000000..3978ef2f8 --- /dev/null +++ b/extern/sector-storage/docs/sector-storage.svg @@ -0,0 +1,3 @@ + + +
LocalWorker
LocalWorker
stores.Local
stores.Local
stores.Store
stores.Store
stores.SectorIndex
stores.SectorInd...
ffiwrapper.Sealer
ffiwrapper.Seal...
SectorProvider
SectorProvider
localProvider
localProvider
Worker
Worker
stores.Remote
stores.Remote
stores.Local
stores.Local
stores.SectorIndex
stores.SectorInd...
localPaths []string
localPaths []str...
urls []string
urls []stri...
stores.SectorIndex
stores.SectorInd...
specs-storage.Prover
specs-storage.Prover
ronlyProvider
ronlyProvider
stores.Index
stores.Index
FetchHandler
FetchHandler
ffiwrapper.Sealer
ffiwrapper.Seal...
SectorProvider
SectorProvider
specs-storage.[Sealer,Storage]
specs-storage.[Sealer,Storage]
specs-storage.Prover
specs-storage.Prover
Manager API
Manager API
Scheduler
Scheduler
[]workerHandle
[]workerHandle
Worker
Worker
WorkerInfo
Worker...
resourceInfo
resourceInfo
schedQueue
schedQueue
stores.SectorIndex
stores.SectorInd...
sector-storage.Manager
sector-storage.Manager
worker management APIs
worker management APIs
Filecoin 'Miner' Node
Filecoin 'Miner' Node
HTTP API
HTTP API
/remote
/remote
JsonRPC
JsonRPC
/rpc/v0
/rpc/v0
LocalWorker
LocalWorker
stores.Local
stores.Local
stores.Store
stores.Store
stores.SectorIndex
stores.SectorInd...
ffiwrapper.Sealer
ffiwrapper.Seal...
SectorProvider
SectorProvider
localProvider
localProvider
Worker
Worker
stores.Remote
stores.Remote
stores.Local
stores.Local
stores.SectorIndex
stores.SectorInd...
localPaths []string
localPaths []str...
urls []string
urls []stri...
stores.SectorIndex
stores.SectorInd...
Miner JsonRPC client
Miner JsonRPC client
miner.Register(remoteWorker)
miner.Register(remoteWorker)
HTTP API
HTTP API
FetchHandler
FetchHandler
/remote
/remote
RemoteWorker
RemoteWorker
/rpc/v0
/rpc/v0
JsonRPC
JsonRPC
Seal Worker Node
Seal Worker NodeViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go new file mode 100644 index 000000000..ea0ab6e01 --- /dev/null +++ b/extern/sector-storage/faults.go @@ -0,0 +1,115 @@ +package sectorstorage + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +// TODO: Track things more actively +type FaultTracker interface { + CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) +} + +// Returns unprovable sectors +func (m *Manager) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, sectors []abi.SectorID) ([]abi.SectorID, error) { + var bad []abi.SectorID + + ssize, err := spt.SectorSize() + if err != nil { + return nil, err + } + + // TODO: More better checks + for _, sector := range sectors { + err := func() error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + locked, err := m.index.StorageTryLock(ctx, sector, stores.FTSealed|stores.FTCache, stores.FTNone) + if err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + if !locked { + log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector, "sealed") + bad = append(bad, sector) + return nil + } + + lp, _, err := m.localStore.AcquireSector(ctx, sector, spt, stores.FTSealed|stores.FTCache, stores.FTNone, stores.PathStorage, stores.AcquireMove) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err) + bad = append(bad, sector) + return nil + } + + if lp.Sealed == "" || lp.Cache == "" { + log.Warnw("CheckProvable Sector FAULT: cache an/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache) + bad = append(bad, sector) + return nil + } + + toCheck := map[string]int64{ + lp.Sealed: 1, + filepath.Join(lp.Cache, "t_aux"): 0, + filepath.Join(lp.Cache, "p_aux"): 0, + } + + addCachePathsForSectorSize(toCheck, lp.Cache, ssize) + + for p, sz := range toCheck { + st, err := os.Stat(p) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err) + bad = append(bad, sector) + return nil + } + + if sz != 0 { + if st.Size() != int64(ssize)*sz { + log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz) + bad = append(bad, sector) + return nil + } + } + } + + return nil + }() + if err != nil { + return nil, err + } + } + + return bad, nil +} + +func addCachePathsForSectorSize(chk map[string]int64, cacheDir string, ssize abi.SectorSize) { + switch ssize { + case 2 << 10: + fallthrough + case 8 << 20: + fallthrough + case 512 << 20: + chk[filepath.Join(cacheDir, "sc-02-data-tree-r-last.dat")] = 0 + case 32 << 30: + for i := 0; i < 8; i++ { + chk[filepath.Join(cacheDir, fmt.Sprintf("sc-02-data-tree-r-last-%d.dat", i))] = 0 + } + case 64 << 30: + for i := 0; i < 16; i++ { + chk[filepath.Join(cacheDir, fmt.Sprintf("sc-02-data-tree-r-last-%d.dat", i))] = 0 + } + default: + log.Warnf("not checking cache files of %s sectors for faults", ssize) + } +} + +var _ FaultTracker = &Manager{} diff --git a/extern/sector-storage/ffiwrapper/basicfs/fs.go b/extern/sector-storage/ffiwrapper/basicfs/fs.go new file mode 100644 index 000000000..3f865f590 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/basicfs/fs.go @@ -0,0 +1,86 @@ +package basicfs + +import ( + "context" + "os" + "path/filepath" + "sync" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +type sectorFile struct { + abi.SectorID + stores.SectorFileType +} + +type Provider struct { + Root string + + lk sync.Mutex + waitSector map[sectorFile]chan struct{} +} + +func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, ptype stores.PathType) (stores.SectorPaths, func(), error) { + if err := os.Mkdir(filepath.Join(b.Root, stores.FTUnsealed.String()), 0755); err != nil && !os.IsExist(err) { + return stores.SectorPaths{}, nil, err + } + if err := os.Mkdir(filepath.Join(b.Root, stores.FTSealed.String()), 0755); err != nil && !os.IsExist(err) { + return stores.SectorPaths{}, nil, err + } + if err := os.Mkdir(filepath.Join(b.Root, stores.FTCache.String()), 0755); err != nil && !os.IsExist(err) { + return stores.SectorPaths{}, nil, err + } + + done := func() {} + + out := stores.SectorPaths{ + Id: id, + } + + for _, fileType := range stores.PathTypes { + if !existing.Has(fileType) && !allocate.Has(fileType) { + continue + } + + b.lk.Lock() + if b.waitSector == nil { + b.waitSector = map[sectorFile]chan struct{}{} + } + ch, found := b.waitSector[sectorFile{id, fileType}] + if !found { + ch = make(chan struct{}, 1) + b.waitSector[sectorFile{id, fileType}] = ch + } + b.lk.Unlock() + + select { + case ch <- struct{}{}: + case <-ctx.Done(): + done() + return stores.SectorPaths{}, nil, ctx.Err() + } + + path := filepath.Join(b.Root, fileType.String(), stores.SectorName(id)) + + prevDone := done + done = func() { + prevDone() + <-ch + } + + if !allocate.Has(fileType) { + if _, err := os.Stat(path); os.IsNotExist(err) { + done() + return stores.SectorPaths{}, nil, storiface.ErrSectorNotFound + } + } + + stores.SetPathByType(&out, fileType, path) + } + + return out, done, nil +} diff --git a/extern/sector-storage/ffiwrapper/config.go b/extern/sector-storage/ffiwrapper/config.go new file mode 100644 index 000000000..707fc6746 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/config.go @@ -0,0 +1,34 @@ +package ffiwrapper + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type Config struct { + SealProofType abi.RegisteredSealProof + + _ struct{} // guard against nameless init +} + +func sizeFromConfig(cfg Config) (abi.SectorSize, error) { + return cfg.SealProofType.SectorSize() +} + +func SealProofTypeFromSectorSize(ssize abi.SectorSize) (abi.RegisteredSealProof, error) { + switch ssize { + case 2 << 10: + return abi.RegisteredSealProof_StackedDrg2KiBV1, nil + case 8 << 20: + return abi.RegisteredSealProof_StackedDrg8MiBV1, nil + case 512 << 20: + return abi.RegisteredSealProof_StackedDrg512MiBV1, nil + case 32 << 30: + return abi.RegisteredSealProof_StackedDrg32GiBV1, nil + case 64 << 30: + return abi.RegisteredSealProof_StackedDrg64GiBV1, nil + default: + return 0, xerrors.Errorf("unsupported sector size for miner: %v", ssize) + } +} diff --git a/extern/sector-storage/ffiwrapper/files.go b/extern/sector-storage/ffiwrapper/files.go new file mode 100644 index 000000000..a13776d2d --- /dev/null +++ b/extern/sector-storage/ffiwrapper/files.go @@ -0,0 +1,53 @@ +package ffiwrapper + +import ( + "io" + "os" + "sync" + + "golang.org/x/xerrors" +) + +func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) { + f, ok := r.(*os.File) + if ok { + return f, func() error { return nil }, nil + } + + var w *os.File + + f, w, err := os.Pipe() + if err != nil { + return nil, nil, err + } + + var wait sync.Mutex + var werr error + + wait.Lock() + go func() { + defer wait.Unlock() + + var copied int64 + copied, werr = io.CopyN(w, r, n) + if werr != nil { + log.Warnf("toReadableFile: copy error: %+v", werr) + } + + err := w.Close() + if werr == nil && err != nil { + werr = err + log.Warnf("toReadableFile: close error: %+v", err) + return + } + if copied != n { + log.Warnf("copied different amount than expected: %d != %d", copied, n) + werr = xerrors.Errorf("copied different amount than expected: %d != %d", copied, n) + } + }() + + return f, func() error { + wait.Lock() + return werr + }, nil +} diff --git a/extern/sector-storage/ffiwrapper/partialfile.go b/extern/sector-storage/ffiwrapper/partialfile.go new file mode 100644 index 000000000..3e8b32288 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/partialfile.go @@ -0,0 +1,316 @@ +package ffiwrapper + +import ( + "encoding/binary" + "io" + "os" + "syscall" + + "github.com/detailyang/go-fallocate" + "golang.org/x/xerrors" + + rlepluslazy "github.com/filecoin-project/go-bitfield/rle" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/filecoin-project/sector-storage/storiface" +) + +const veryLargeRle = 1 << 20 + +// Sectors can be partially unsealed. We support this by appending a small +// trailer to each unsealed sector file containing an RLE+ marking which bytes +// in a sector are unsealed, and which are not (holes) + +// unsealed sector files internally have this structure +// [unpadded (raw) data][rle+][4B LE length fo the rle+ field] + +type partialFile struct { + maxPiece abi.PaddedPieceSize + + path string + allocated rlepluslazy.RLE + + file *os.File +} + +func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) error { + trailer, err := rlepluslazy.EncodeRuns(r, nil) + if err != nil { + return xerrors.Errorf("encoding trailer: %w", err) + } + + // maxPieceSize == unpadded(sectorSize) == trailer start + if _, err := w.Seek(maxPieceSize, io.SeekStart); err != nil { + return xerrors.Errorf("seek to trailer start: %w", err) + } + + rb, err := w.Write(trailer) + if err != nil { + return xerrors.Errorf("writing trailer data: %w", err) + } + + if err := binary.Write(w, binary.LittleEndian, uint32(len(trailer))); err != nil { + return xerrors.Errorf("writing trailer length: %w", err) + } + + return w.Truncate(maxPieceSize + int64(rb) + 4) +} + +func createPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) + } + + err = func() error { + err := fallocate.Fallocate(f, 0, int64(maxPieceSize)) + if errno, ok := err.(syscall.Errno); ok { + if errno == syscall.EOPNOTSUPP || errno == syscall.ENOSYS { + log.Warnf("could not allocated space, ignoring: %v", errno) + err = nil // log and ignore + } + } + if err != nil { + return xerrors.Errorf("fallocate '%s': %w", path, err) + } + + if err := writeTrailer(int64(maxPieceSize), f, &rlepluslazy.RunSliceIterator{}); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil + }() + if err != nil { + f.Close() + return nil, err + } + if err := f.Close(); err != nil { + return nil, xerrors.Errorf("close empty partial file: %w", err) + } + + return openPartialFile(maxPieceSize, path) +} + +func openPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialFile, error) { + f, err := os.OpenFile(path, os.O_RDWR, 0644) + if err != nil { + return nil, xerrors.Errorf("openning partial file '%s': %w", path, err) + } + + var rle rlepluslazy.RLE + err = func() error { + st, err := f.Stat() + if err != nil { + return xerrors.Errorf("stat '%s': %w", path, err) + } + if st.Size() < int64(maxPieceSize) { + return xerrors.Errorf("sector file '%s' was smaller than the sector size %d < %d", path, st.Size(), maxPieceSize) + } + // read trailer + var tlen [4]byte + _, err = f.ReadAt(tlen[:], st.Size()-int64(len(tlen))) + if err != nil { + return xerrors.Errorf("reading trailer length: %w", err) + } + + // sanity-check the length + trailerLen := binary.LittleEndian.Uint32(tlen[:]) + expectLen := int64(trailerLen) + int64(len(tlen)) + int64(maxPieceSize) + if expectLen != st.Size() { + return xerrors.Errorf("file '%d' has inconsistent length; has %d bytes; expected %d (%d trailer, %d sector data)", path, st.Size(), expectLen, int64(trailerLen)+int64(len(tlen)), maxPieceSize) + } + if trailerLen > veryLargeRle { + log.Warnf("Partial file '%s' has a VERY large trailer with %d bytes", path, trailerLen) + } + + trailerStart := st.Size() - int64(len(tlen)) - int64(trailerLen) + if trailerStart != int64(maxPieceSize) { + return xerrors.Errorf("expected sector size to equal trailer start index") + } + + trailerBytes := make([]byte, trailerLen) + _, err = f.ReadAt(trailerBytes, trailerStart) + if err != nil { + return xerrors.Errorf("reading trailer: %w", err) + } + + rle, err = rlepluslazy.FromBuf(trailerBytes) + if err != nil { + return xerrors.Errorf("decoding trailer: %w", err) + } + + it, err := rle.RunIterator() + if err != nil { + return xerrors.Errorf("getting trailer run iterator: %w", err) + } + + f, err := rlepluslazy.Fill(it) + if err != nil { + return xerrors.Errorf("filling bitfield: %w", err) + } + lastSet, err := rlepluslazy.Count(f) + if err != nil { + return xerrors.Errorf("finding last set byte index: %w", err) + } + + if lastSet > uint64(maxPieceSize) { + return xerrors.Errorf("last set byte at index higher than sector size: %d > %d", lastSet, maxPieceSize) + } + + return nil + }() + if err != nil { + f.Close() + return nil, err + } + + return &partialFile{ + maxPiece: maxPieceSize, + path: path, + allocated: rle, + file: f, + }, nil +} + +func (pf *partialFile) Close() error { + return pf.file.Close() +} + +func (pf *partialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { + if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek piece start: %w", err) + } + + { + have, err := pf.allocated.RunIterator() + if err != nil { + return nil, err + } + + and, err := rlepluslazy.And(have, pieceRun(offset, size)) + if err != nil { + return nil, err + } + + c, err := rlepluslazy.Count(and) + if err != nil { + return nil, err + } + + if c > 0 { + log.Warnf("getting partial file writer overwriting %d allocated bytes", c) + } + } + + return pf.file, nil +} + +func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { + have, err := pf.allocated.RunIterator() + if err != nil { + return err + } + + ored, err := rlepluslazy.Or(have, pieceRun(offset, size)) + if err != nil { + return err + } + + if err := writeTrailer(int64(pf.maxPiece), pf.file, ored); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil +} + +func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { + have, err := pf.allocated.RunIterator() + if err != nil { + return err + } + + if err := fsutil.Deallocate(pf.file, int64(offset), int64(size)); err != nil { + return xerrors.Errorf("deallocating: %w", err) + } + + s, err := rlepluslazy.Subtract(have, pieceRun(offset, size)) + if err != nil { + return err + } + + if err := writeTrailer(int64(pf.maxPiece), pf.file, s); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil +} + +func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { + if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek piece start: %w", err) + } + + { + have, err := pf.allocated.RunIterator() + if err != nil { + return nil, err + } + + and, err := rlepluslazy.And(have, pieceRun(offset, size)) + if err != nil { + return nil, err + } + + c, err := rlepluslazy.Count(and) + if err != nil { + return nil, err + } + + if c != uint64(size) { + log.Warnf("getting partial file reader reading %d unallocated bytes", uint64(size)-c) + } + } + + return pf.file, nil +} + +func (pf *partialFile) Allocated() (rlepluslazy.RunIterator, error) { + return pf.allocated.RunIterator() +} + +func (pf *partialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + have, err := pf.Allocated() + if err != nil { + return false, err + } + + u, err := rlepluslazy.And(have, pieceRun(offset.Padded(), size.Padded())) + if err != nil { + return false, err + } + + uc, err := rlepluslazy.Count(u) + if err != nil { + return false, err + } + + return abi.PaddedPieceSize(uc) == size.Padded(), nil +} + +func pieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { + var runs []rlepluslazy.Run + if offset > 0 { + runs = append(runs, rlepluslazy.Run{ + Val: false, + Len: uint64(offset), + }) + } + + runs = append(runs, rlepluslazy.Run{ + Val: true, + Len: uint64(size), + }) + + return &rlepluslazy.RunSliceIterator{Runs: runs} +} diff --git a/extern/sector-storage/ffiwrapper/sealer.go b/extern/sector-storage/ffiwrapper/sealer.go new file mode 100644 index 000000000..c97557a37 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/sealer.go @@ -0,0 +1,28 @@ +package ffiwrapper + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("ffiwrapper") + +type Sealer struct { + sealProofType abi.RegisteredSealProof + ssize abi.SectorSize // a function of sealProofType and postProofType + + sectors SectorProvider + stopping chan struct{} +} + +func (sb *Sealer) Stop() { + close(sb.stopping) +} + +func (sb *Sealer) SectorSize() abi.SectorSize { + return sb.ssize +} + +func (sb *Sealer) SealProofType() abi.RegisteredSealProof { + return sb.sealProofType +} diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go new file mode 100644 index 000000000..8a4f18bc7 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -0,0 +1,673 @@ +//+build cgo + +package ffiwrapper + +import ( + "bufio" + "bytes" + "context" + "io" + "math/bits" + "os" + "runtime" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + ffi "github.com/filecoin-project/filecoin-ffi" + rlepluslazy "github.com/filecoin-project/go-bitfield/rle" + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/fr32" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" + "github.com/filecoin-project/sector-storage/zerocomm" +) + +var _ Storage = &Sealer{} + +func New(sectors SectorProvider, cfg *Config) (*Sealer, error) { + sectorSize, err := sizeFromConfig(*cfg) + if err != nil { + return nil, err + } + + sb := &Sealer{ + sealProofType: cfg.SealProofType, + ssize: sectorSize, + + sectors: sectors, + + stopping: make(chan struct{}), + } + + return sb, nil +} + +func (sb *Sealer) NewSector(ctx context.Context, sector abi.SectorID) error { + // TODO: Allocate the sector here instead of in addpiece + + return nil +} + +func (sb *Sealer) AddPiece(ctx context.Context, sector abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error) { + var offset abi.UnpaddedPieceSize + for _, size := range existingPieceSizes { + offset += size + } + + maxPieceSize := abi.PaddedPieceSize(sb.ssize) + + if offset.Padded()+pieceSize.Padded() > maxPieceSize { + return abi.PieceInfo{}, xerrors.Errorf("can't add %d byte piece to sector %v with %d bytes of existing pieces", pieceSize, sector, offset) + } + + var err error + var done func() + var stagedFile *partialFile + + defer func() { + if done != nil { + done() + } + + if stagedFile != nil { + if err := stagedFile.Close(); err != nil { + log.Errorf("closing staged file: %+v", err) + } + } + }() + + var stagedPath stores.SectorPaths + if len(existingPieceSizes) == 0 { + stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, 0, stores.FTUnsealed, stores.PathSealing) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) + } + + stagedFile, err = createPartialFile(maxPieceSize, stagedPath.Unsealed) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err) + } + } else { + stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, stores.PathSealing) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) + } + + stagedFile, err = openPartialFile(maxPieceSize, stagedPath.Unsealed) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("opening unsealed sector file: %w", err) + } + } + + w, err := stagedFile.Writer(storiface.UnpaddedByteIndex(offset).Padded(), pieceSize.Padded()) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("getting partial file writer: %w", err) + } + + pw := fr32.NewPadWriter(w) + + pr := io.TeeReader(io.LimitReader(file, int64(pieceSize)), pw) + + chunk := abi.PaddedPieceSize(4 << 20) + + buf := make([]byte, chunk.Unpadded()) + var pieceCids []abi.PieceInfo + + for { + var read int + for rbuf := buf; len(rbuf) > 0; { + n, err := pr.Read(rbuf) + if err != nil && err != io.EOF { + return abi.PieceInfo{}, xerrors.Errorf("pr read error: %w", err) + } + + rbuf = rbuf[n:] + read += n + + if err == io.EOF { + break + } + } + if read == 0 { + break + } + + c, err := sb.pieceCid(buf[:read]) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("pieceCid error: %w", err) + } + pieceCids = append(pieceCids, abi.PieceInfo{ + Size: abi.UnpaddedPieceSize(len(buf[:read])).Padded(), + PieceCID: c, + }) + } + + if err := pw.Close(); err != nil { + return abi.PieceInfo{}, xerrors.Errorf("closing padded writer: %w", err) + } + + if err := stagedFile.MarkAllocated(storiface.UnpaddedByteIndex(offset).Padded(), pieceSize.Padded()); err != nil { + return abi.PieceInfo{}, xerrors.Errorf("marking data range as allocated: %w", err) + } + + if err := stagedFile.Close(); err != nil { + return abi.PieceInfo{}, err + } + stagedFile = nil + + if len(pieceCids) == 1 { + return pieceCids[0], nil + } + + pieceCID, err := ffi.GenerateUnsealedCID(sb.sealProofType, pieceCids) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("generate unsealed CID: %w", err) + } + + // validate that the pieceCID was properly formed + if _, err := commcid.CIDToPieceCommitmentV1(pieceCID); err != nil { + return abi.PieceInfo{}, err + } + + return abi.PieceInfo{ + Size: pieceSize.Padded(), + PieceCID: pieceCID, + }, nil +} + +func (sb *Sealer) pieceCid(in []byte) (cid.Cid, error) { + prf, werr, err := ToReadableFile(bytes.NewReader(in), int64(len(in))) + if err != nil { + return cid.Undef, xerrors.Errorf("getting tee reader pipe: %w", err) + } + + pieceCID, err := ffi.GeneratePieceCIDFromFile(sb.sealProofType, prf, abi.UnpaddedPieceSize(len(in))) + if err != nil { + return cid.Undef, xerrors.Errorf("generating piece commitment: %w", err) + } + + prf.Close() + + return pieceCID, werr() +} + +func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error { + maxPieceSize := abi.PaddedPieceSize(sb.ssize) + + // try finding existing + unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, stores.PathStorage) + var pf *partialFile + + switch { + case xerrors.Is(err, storiface.ErrSectorNotFound): + unsealedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTNone, stores.FTUnsealed, stores.PathStorage) + if err != nil { + return xerrors.Errorf("acquire unsealed sector path (allocate): %w", err) + } + defer done() + + pf, err = createPartialFile(maxPieceSize, unsealedPath.Unsealed) + if err != nil { + return xerrors.Errorf("create unsealed file: %w", err) + } + + case err == nil: + defer done() + + pf, err = openPartialFile(maxPieceSize, unsealedPath.Unsealed) + if err != nil { + return xerrors.Errorf("opening partial file: %w", err) + } + default: + return xerrors.Errorf("acquire unsealed sector path (existing): %w", err) + } + defer pf.Close() + + allocated, err := pf.Allocated() + if err != nil { + return xerrors.Errorf("getting bitruns of allocated data: %w", err) + } + + toUnseal, err := computeUnsealRanges(allocated, offset, size) + if err != nil { + return xerrors.Errorf("computing unseal ranges: %w", err) + } + + if !toUnseal.HasNext() { + return nil + } + + srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache|stores.FTSealed, stores.FTNone, stores.PathStorage) + if err != nil { + return xerrors.Errorf("acquire sealed sector paths: %w", err) + } + defer srcDone() + + sealed, err := os.OpenFile(srcPaths.Sealed, os.O_RDONLY, 0644) + if err != nil { + return xerrors.Errorf("opening sealed file: %w", err) + } + defer sealed.Close() + + var at, nextat abi.PaddedPieceSize + first := true + for first || toUnseal.HasNext() { + first = false + + piece, err := toUnseal.NextRun() + if err != nil { + return xerrors.Errorf("getting next range to unseal: %w", err) + } + + at = nextat + nextat += abi.PaddedPieceSize(piece.Len) + + if !piece.Val { + continue + } + + out, err := pf.Writer(offset.Padded(), size.Padded()) + if err != nil { + return xerrors.Errorf("getting partial file writer: %w", err) + } + + // + opr, opw, err := os.Pipe() + if err != nil { + return xerrors.Errorf("creating out pipe: %w", err) + } + + var perr error + outWait := make(chan struct{}) + + { + go func() { + defer close(outWait) + defer opr.Close() + + padwriter := fr32.NewPadWriter(out) + if err != nil { + perr = xerrors.Errorf("creating new padded writer: %w", err) + return + } + + bsize := uint64(size.Padded()) + if bsize > uint64(runtime.NumCPU())*fr32.MTTresh { + bsize = uint64(runtime.NumCPU()) * fr32.MTTresh + } + + bw := bufio.NewWriterSize(padwriter, int(abi.PaddedPieceSize(bsize).Unpadded())) + + _, err = io.CopyN(bw, opr, int64(size)) + if err != nil { + perr = xerrors.Errorf("copying data: %w", err) + return + } + + if err := bw.Flush(); err != nil { + perr = xerrors.Errorf("flushing unpadded data: %w", err) + return + } + + if err := padwriter.Close(); err != nil { + perr = xerrors.Errorf("closing padwriter: %w", err) + return + } + }() + } + // + + // TODO: This may be possible to do in parallel + err = ffi.UnsealRange(sb.sealProofType, + srcPaths.Cache, + sealed, + opw, + sector.Number, + sector.Miner, + randomness, + commd, + uint64(at.Unpadded()), + uint64(abi.PaddedPieceSize(piece.Len).Unpadded())) + + _ = opw.Close() + + if err != nil { + return xerrors.Errorf("unseal range: %w", err) + } + + select { + case <-outWait: + case <-ctx.Done(): + return ctx.Err() + } + + if perr != nil { + return xerrors.Errorf("piping output to unsealed file: %w", perr) + } + + if err := pf.MarkAllocated(storiface.PaddedByteIndex(at), abi.PaddedPieceSize(piece.Len)); err != nil { + return xerrors.Errorf("marking unsealed range as allocated: %w", err) + } + + if !toUnseal.HasNext() { + break + } + } + + return nil +} + +func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + path, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, stores.PathStorage) + if err != nil { + return false, xerrors.Errorf("acquire unsealed sector path: %w", err) + } + defer done() + + maxPieceSize := abi.PaddedPieceSize(sb.ssize) + + pf, err := openPartialFile(maxPieceSize, path.Unsealed) + if xerrors.Is(err, os.ErrNotExist) { + return false, xerrors.Errorf("opening partial file: %w", err) + } + + ok, err := pf.HasAllocated(offset, size) + if err != nil { + pf.Close() + return false, err + } + + if !ok { + pf.Close() + return false, nil + } + + f, err := pf.Reader(offset.Padded(), size.Padded()) + if err != nil { + pf.Close() + return false, xerrors.Errorf("getting partial file reader: %w", err) + } + + upr, err := fr32.NewUnpadReader(f, size.Padded()) + if err != nil { + return false, xerrors.Errorf("creating unpadded reader: %w", err) + } + + if _, err := io.CopyN(writer, upr, int64(size)); err != nil { + pf.Close() + return false, xerrors.Errorf("reading unsealed file: %w", err) + } + + if err := pf.Close(); err != nil { + return false, xerrors.Errorf("closing partial file: %w", err) + } + + return false, nil +} + +func (sb *Sealer) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache, stores.PathSealing) + if err != nil { + return nil, xerrors.Errorf("acquiring sector paths: %w", err) + } + defer done() + + e, err := os.OpenFile(paths.Sealed, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, xerrors.Errorf("ensuring sealed file exists: %w", err) + } + if err := e.Close(); err != nil { + return nil, err + } + + if err := os.Mkdir(paths.Cache, 0755); err != nil { + if os.IsExist(err) { + log.Warnf("existing cache in %s; removing", paths.Cache) + + if err := os.RemoveAll(paths.Cache); err != nil { + return nil, xerrors.Errorf("remove existing sector cache from %s (sector %d): %w", paths.Cache, sector, err) + } + + if err := os.Mkdir(paths.Cache, 0755); err != nil { + return nil, xerrors.Errorf("mkdir cache path after cleanup: %w", err) + } + } else { + return nil, err + } + } + + var sum abi.UnpaddedPieceSize + for _, piece := range pieces { + sum += piece.Size.Unpadded() + } + ussize := abi.PaddedPieceSize(sb.ssize).Unpadded() + if sum != ussize { + return nil, xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum)) + } + + // TODO: context cancellation respect + p1o, err := ffi.SealPreCommitPhase1( + sb.sealProofType, + paths.Cache, + paths.Unsealed, + paths.Sealed, + sector.Number, + sector.Miner, + ticket, + pieces, + ) + if err != nil { + return nil, xerrors.Errorf("presealing sector %d (%s): %w", sector.Number, paths.Unsealed, err) + } + return p1o, nil +} + +func (sb *Sealer) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, stores.PathSealing) + if err != nil { + return storage.SectorCids{}, xerrors.Errorf("acquiring sector paths: %w", err) + } + defer done() + + sealedCID, unsealedCID, err := ffi.SealPreCommitPhase2(phase1Out, paths.Cache, paths.Sealed) + if err != nil { + return storage.SectorCids{}, xerrors.Errorf("presealing sector %d (%s): %w", sector.Number, paths.Unsealed, err) + } + + return storage.SectorCids{ + Unsealed: unsealedCID, + Sealed: sealedCID, + }, nil +} + +func (sb *Sealer) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, stores.PathSealing) + if err != nil { + return nil, xerrors.Errorf("acquire sector paths: %w", err) + } + defer done() + output, err := ffi.SealCommitPhase1( + sb.sealProofType, + cids.Sealed, + cids.Unsealed, + paths.Cache, + paths.Sealed, + sector.Number, + sector.Miner, + ticket, + seed, + pieces, + ) + if err != nil { + log.Warn("StandaloneSealCommit error: ", err) + log.Warnf("num:%d tkt:%v seed:%v, pi:%v sealedCID:%v, unsealedCID:%v", sector.Number, ticket, seed, pieces, cids.Sealed, cids.Unsealed) + + return nil, xerrors.Errorf("StandaloneSealCommit: %w", err) + } + return output, nil +} + +func (sb *Sealer) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (storage.Proof, error) { + return ffi.SealCommitPhase2(phase1Out, sector.Number, sector.Miner) +} + +func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { + if len(keepUnsealed) > 0 { + maxPieceSize := abi.PaddedPieceSize(sb.ssize) + + sr := pieceRun(0, maxPieceSize) + + for _, s := range keepUnsealed { + si := &rlepluslazy.RunSliceIterator{} + if s.Offset != 0 { + si.Runs = append(si.Runs, rlepluslazy.Run{Val: false, Len: uint64(s.Offset)}) + } + si.Runs = append(si.Runs, rlepluslazy.Run{Val: true, Len: uint64(s.Size)}) + + var err error + sr, err = rlepluslazy.Subtract(sr, si) + if err != nil { + return err + } + } + + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, stores.PathStorage) + if err != nil { + return xerrors.Errorf("acquiring sector cache path: %w", err) + } + defer done() + + pf, err := openPartialFile(maxPieceSize, paths.Unsealed) + if xerrors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("opening partial file: %w", err) + } + + var at uint64 + for sr.HasNext() { + r, err := sr.NextRun() + if err != nil { + _ = pf.Close() + return err + } + + offset := at + at += r.Len + if !r.Val { + continue + } + + err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) + if err != nil { + _ = pf.Close() + return xerrors.Errorf("free partial file range: %w", err) + } + } + + if err := pf.Close(); err != nil { + return err + } + } + + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, stores.PathStorage) + if err != nil { + return xerrors.Errorf("acquiring sector cache path: %w", err) + } + defer done() + + return ffi.ClearCache(uint64(sb.ssize), paths.Cache) +} + +func (sb *Sealer) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { + // This call is meant to mark storage as 'freeable'. Given that unsealing is + // very expensive, we don't remove data as soon as we can - instead we only + // do that when we don't have free space for data that really needs it + + // This function should not be called at this layer, everything should be + // handled in localworker + return xerrors.Errorf("not supported at this layer") +} + +func (sb *Sealer) Remove(ctx context.Context, sector abi.SectorID) error { + return xerrors.Errorf("not supported at this layer") // happens in localworker +} + +func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader, pieceSize abi.UnpaddedPieceSize) (cid.Cid, error) { + f, werr, err := ToReadableFile(piece, int64(pieceSize)) + if err != nil { + return cid.Undef, err + } + + pieceCID, err := ffi.GeneratePieceCIDFromFile(proofType, f, pieceSize) + if err != nil { + return cid.Undef, err + } + + return pieceCID, werr() +} + +func GetRequiredPadding(oldLength abi.PaddedPieceSize, newPieceLength abi.PaddedPieceSize) ([]abi.PaddedPieceSize, abi.PaddedPieceSize) { + + padPieces := make([]abi.PaddedPieceSize, 0) + + toFill := uint64(-oldLength % newPieceLength) + + n := bits.OnesCount64(toFill) + var sum abi.PaddedPieceSize + for i := 0; i < n; i++ { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << uint(next) + toFill ^= psize + + padded := abi.PaddedPieceSize(psize) + padPieces = append(padPieces, padded) + sum += padded + } + + return padPieces, sum +} + +func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { + ssize, err := proofType.SectorSize() + if err != nil { + return cid.Undef, err + } + + pssize := abi.PaddedPieceSize(ssize) + allPieces := make([]abi.PieceInfo, 0, len(pieces)) + if len(pieces) == 0 { + allPieces = append(allPieces, abi.PieceInfo{ + Size: pssize, + PieceCID: zerocomm.ZeroPieceCommitment(pssize.Unpadded()), + }) + } else { + var sum abi.PaddedPieceSize + + padTo := func(pads []abi.PaddedPieceSize) { + for _, p := range pads { + allPieces = append(allPieces, abi.PieceInfo{ + Size: p, + PieceCID: zerocomm.ZeroPieceCommitment(p.Unpadded()), + }) + + sum += p + } + } + + for _, p := range pieces { + ps, _ := GetRequiredPadding(sum, p.Size) + padTo(ps) + + allPieces = append(allPieces, p) + sum += p.Size + } + + ps, _ := GetRequiredPadding(sum, pssize) + padTo(ps) + } + + return ffi.GenerateUnsealedCID(proofType, allPieces) +} diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go new file mode 100644 index 000000000..f795be159 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -0,0 +1,596 @@ +package ffiwrapper + +import ( + "bytes" + "context" + "fmt" + "github.com/ipfs/go-cid" + "io" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + logging "github.com/ipfs/go-log" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + ffi "github.com/filecoin-project/filecoin-ffi" + paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/ffiwrapper/basicfs" + "github.com/filecoin-project/sector-storage/stores" +) + +func init() { + logging.SetLogLevel("*", "DEBUG") //nolint: errcheck +} + +var sealProofType = abi.RegisteredSealProof_StackedDrg2KiBV1 +var sectorSize, _ = sealProofType.SectorSize() + +var sealRand = abi.SealRandomness{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2} + +type seal struct { + id abi.SectorID + cids storage.SectorCids + pi abi.PieceInfo + ticket abi.SealRandomness +} + +func data(sn abi.SectorNumber, dlen abi.UnpaddedPieceSize) io.Reader { + return io.MultiReader( + io.LimitReader(rand.New(rand.NewSource(42+int64(sn))), int64(123)), + io.LimitReader(rand.New(rand.NewSource(42+int64(sn))), int64(dlen-123)), + ) +} + +func (s *seal) precommit(t *testing.T, sb *Sealer, id abi.SectorID, done func()) { + defer done() + dlen := abi.PaddedPieceSize(sectorSize).Unpadded() + + var err error + r := data(id.Number, dlen) + s.pi, err = sb.AddPiece(context.TODO(), id, []abi.UnpaddedPieceSize{}, dlen, r) + if err != nil { + t.Fatalf("%+v", err) + } + + s.ticket = sealRand + + p1, err := sb.SealPreCommit1(context.TODO(), id, s.ticket, []abi.PieceInfo{s.pi}) + if err != nil { + t.Fatalf("%+v", err) + } + cids, err := sb.SealPreCommit2(context.TODO(), id, p1) + if err != nil { + t.Fatalf("%+v", err) + } + s.cids = cids +} + +func (s *seal) commit(t *testing.T, sb *Sealer, done func()) { + defer done() + seed := abi.InteractiveSealRandomness{0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9} + + pc1, err := sb.SealCommit1(context.TODO(), s.id, s.ticket, seed, []abi.PieceInfo{s.pi}, s.cids) + if err != nil { + t.Fatalf("%+v", err) + } + proof, err := sb.SealCommit2(context.TODO(), s.id, pc1) + if err != nil { + t.Fatalf("%+v", err) + } + + ok, err := ProofVerifier.VerifySeal(abi.SealVerifyInfo{ + SectorID: s.id, + SealedCID: s.cids.Sealed, + SealProof: sealProofType, + Proof: proof, + Randomness: s.ticket, + InteractiveRandomness: seed, + UnsealedCID: s.cids.Unsealed, + }) + if err != nil { + t.Fatalf("%+v", err) + } + + if !ok { + t.Fatal("proof failed to validate") + } +} + +func (s *seal) unseal(t *testing.T, sb *Sealer, sp *basicfs.Provider, si abi.SectorID, done func()) { + defer done() + + var b bytes.Buffer + _, err := sb.ReadPiece(context.TODO(), &b, si, 0, 1016) + if err != nil { + t.Fatal(err) + } + + expect, _ := ioutil.ReadAll(data(si.Number, 1016)) + if !bytes.Equal(b.Bytes(), expect) { + t.Fatal("read wrong bytes") + } + + p, sd, err := sp.AcquireSector(context.TODO(), si, stores.FTUnsealed, stores.FTNone, stores.PathStorage) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(p.Unsealed); err != nil { + t.Fatal(err) + } + sd() + + _, err = sb.ReadPiece(context.TODO(), &b, si, 0, 1016) + if err == nil { + t.Fatal("HOW?!") + } + log.Info("this is what we expect: ", err) + + if err := sb.UnsealPiece(context.TODO(), si, 0, 1016, sealRand, s.cids.Unsealed); err != nil { + t.Fatal(err) + } + + b.Reset() + _, err = sb.ReadPiece(context.TODO(), &b, si, 0, 1016) + if err != nil { + t.Fatal(err) + } + + expect, _ = ioutil.ReadAll(data(si.Number, 1016)) + require.Equal(t, expect, b.Bytes()) + + b.Reset() + have, err := sb.ReadPiece(context.TODO(), &b, si, 0, 2032) + if err != nil { + t.Fatal(err) + } + + if have { + t.Errorf("didn't expect to read things") + } + + if b.Len() != 0 { + t.Fatal("read bytes") + } +} + +func post(t *testing.T, sealer *Sealer, seals ...seal) time.Time { + /*randomness := abi.PoStRandomness{0, 9, 2, 7, 6, 5, 4, 3, 2, 1, 0, 9, 8, 7, 6, 45, 3, 2, 1, 0, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 7} + + sis := make([]abi.SectorInfo, len(seals)) + for i, s := range seals { + sis[i] = abi.SectorInfo{ + RegisteredProof: sealProofType, + SectorNumber: s.id.Number, + SealedCID: s.cids.Sealed, + } + } + + candidates, err := sealer.GenerateEPostCandidates(context.TODO(), seals[0].id.Miner, sis, randomness, []abi.SectorNumber{}) + if err != nil { + t.Fatalf("%+v", err) + }*/ + + fmt.Println("skipping post") + + genCandidates := time.Now() + + /*if len(candidates) != 1 { + t.Fatal("expected 1 candidate") + } + + candidatesPrime := make([]abi.PoStCandidate, len(candidates)) + for idx := range candidatesPrime { + candidatesPrime[idx] = candidates[idx].Candidate + } + + proofs, err := sealer.ComputeElectionPoSt(context.TODO(), seals[0].id.Miner, sis, randomness, candidatesPrime) + if err != nil { + t.Fatalf("%+v", err) + } + + ePoStChallengeCount := ElectionPostChallengeCount(uint64(len(sis)), 0) + + ok, err := ProofVerifier.VerifyElectionPost(context.TODO(), abi.PoStVerifyInfo{ + Randomness: randomness, + Candidates: candidatesPrime, + Proofs: proofs, + EligibleSectors: sis, + Prover: seals[0].id.Miner, + ChallengeCount: ePoStChallengeCount, + }) + if err != nil { + t.Fatalf("%+v", err) + } + if !ok { + t.Fatal("bad post") + } + */ + return genCandidates +} + +func getGrothParamFileAndVerifyingKeys(s abi.SectorSize) { + dat, err := ioutil.ReadFile("../parameters.json") + if err != nil { + panic(err) + } + + err = paramfetch.GetParams(dat, uint64(s)) + if err != nil { + panic(xerrors.Errorf("failed to acquire Groth parameters for 2KiB sectors: %w", err)) + } +} + +// TestDownloadParams exists only so that developers and CI can pre-download +// Groth parameters and verifying keys before running the tests which rely on +// those parameters and keys. To do this, run the following command: +// +// go test -run=^TestDownloadParams +// +func TestDownloadParams(t *testing.T) { + defer requireFDsClosed(t, openFDs(t)) + + getGrothParamFileAndVerifyingKeys(sectorSize) +} + +func TestSealAndVerify(t *testing.T) { + defer requireFDsClosed(t, openFDs(t)) + + if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware + t.Skip("this is slow") + } + _ = os.Setenv("RUST_LOG", "info") + + getGrothParamFileAndVerifyingKeys(sectorSize) + + cdir, err := ioutil.TempDir("", "sbtest-c-") + if err != nil { + t.Fatal(err) + } + miner := abi.ActorID(123) + + cfg := &Config{ + SealProofType: sealProofType, + } + + sp := &basicfs.Provider{ + Root: cdir, + } + sb, err := New(sp, cfg) + if err != nil { + t.Fatalf("%+v", err) + } + cleanup := func() { + if t.Failed() { + fmt.Printf("not removing %s\n", cdir) + return + } + if err := os.RemoveAll(cdir); err != nil { + t.Error(err) + } + } + defer cleanup() + + si := abi.SectorID{Miner: miner, Number: 1} + + s := seal{id: si} + + start := time.Now() + + s.precommit(t, sb, si, func() {}) + + precommit := time.Now() + + s.commit(t, sb, func() {}) + + commit := time.Now() + + genCandidiates := post(t, sb, s) + + epost := time.Now() + + post(t, sb, s) + + if err := sb.FinalizeSector(context.TODO(), si, nil); err != nil { + t.Fatalf("%+v", err) + } + + s.unseal(t, sb, sp, si, func() {}) + + fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) + fmt.Printf("Commit: %s\n", commit.Sub(precommit).String()) + fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(commit).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) +} + +func TestSealPoStNoCommit(t *testing.T) { + defer requireFDsClosed(t, openFDs(t)) + + if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware + t.Skip("this is slow") + } + _ = os.Setenv("RUST_LOG", "info") + + getGrothParamFileAndVerifyingKeys(sectorSize) + + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + t.Fatal(err) + } + + miner := abi.ActorID(123) + + cfg := &Config{ + SealProofType: sealProofType, + } + sp := &basicfs.Provider{ + Root: dir, + } + sb, err := New(sp, cfg) + if err != nil { + t.Fatalf("%+v", err) + } + + cleanup := func() { + if t.Failed() { + fmt.Printf("not removing %s\n", dir) + return + } + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + } + defer cleanup() + + si := abi.SectorID{Miner: miner, Number: 1} + + s := seal{id: si} + + start := time.Now() + + s.precommit(t, sb, si, func() {}) + + precommit := time.Now() + + if err := sb.FinalizeSector(context.TODO(), si, nil); err != nil { + t.Fatal(err) + } + + genCandidiates := post(t, sb, s) + + epost := time.Now() + + fmt.Printf("PreCommit: %s\n", precommit.Sub(start).String()) + fmt.Printf("GenCandidates: %s\n", genCandidiates.Sub(precommit).String()) + fmt.Printf("EPoSt: %s\n", epost.Sub(genCandidiates).String()) +} + +func TestSealAndVerify2(t *testing.T) { + defer requireFDsClosed(t, openFDs(t)) + + if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware + t.Skip("this is slow") + } + _ = os.Setenv("RUST_LOG", "trace") + + getGrothParamFileAndVerifyingKeys(sectorSize) + + dir, err := ioutil.TempDir("", "sbtest") + if err != nil { + t.Fatal(err) + } + + miner := abi.ActorID(123) + + cfg := &Config{ + SealProofType: sealProofType, + } + sp := &basicfs.Provider{ + Root: dir, + } + sb, err := New(sp, cfg) + if err != nil { + t.Fatalf("%+v", err) + } + + cleanup := func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + } + + defer cleanup() + + var wg sync.WaitGroup + + si1 := abi.SectorID{Miner: miner, Number: 1} + si2 := abi.SectorID{Miner: miner, Number: 2} + + s1 := seal{id: si1} + s2 := seal{id: si2} + + wg.Add(2) + go s1.precommit(t, sb, si1, wg.Done) //nolint: staticcheck + time.Sleep(100 * time.Millisecond) + go s2.precommit(t, sb, si2, wg.Done) //nolint: staticcheck + wg.Wait() + + wg.Add(2) + go s1.commit(t, sb, wg.Done) //nolint: staticcheck + go s2.commit(t, sb, wg.Done) //nolint: staticcheck + wg.Wait() + + post(t, sb, s1, s2) +} + +func BenchmarkWriteWithAlignment(b *testing.B) { + bt := abi.UnpaddedPieceSize(2 * 127 * 1024 * 1024) + b.SetBytes(int64(bt)) + + for i := 0; i < b.N; i++ { + b.StopTimer() + rf, w, _ := ToReadableFile(bytes.NewReader(bytes.Repeat([]byte{0xff, 0}, int(bt/2))), int64(bt)) + tf, _ := ioutil.TempFile("/tmp/", "scrb-") + b.StartTimer() + + ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg2KiBV1, rf, bt, tf, nil) + w() + } +} + +func openFDs(t *testing.T) int { + dent, err := ioutil.ReadDir("/proc/self/fd") + require.NoError(t, err) + + var skip int + for _, info := range dent { + l, err := os.Readlink(filepath.Join("/proc/self/fd", info.Name())) + if err != nil { + continue + } + + if strings.HasPrefix(l, "/dev/nvidia") { + skip++ + } + + if strings.HasPrefix(l, "/var/tmp/filecoin-proof-parameters/") { + skip++ + } + } + + return len(dent) - skip +} + +func requireFDsClosed(t *testing.T, start int) { + openNow := openFDs(t) + + if start != openNow { + dent, err := ioutil.ReadDir("/proc/self/fd") + require.NoError(t, err) + + for _, info := range dent { + l, err := os.Readlink(filepath.Join("/proc/self/fd", info.Name())) + if err != nil { + fmt.Printf("FD err %s\n", err) + continue + } + + fmt.Printf("FD %s -> %s\n", info.Name(), l) + } + } + + log.Infow("open FDs", "start", start, "now", openNow) + require.Equal(t, start, openNow, "FDs shouldn't leak") +} + +func TestGenerateUnsealedCID(t *testing.T) { + pt := abi.RegisteredSealProof_StackedDrg2KiBV1 + ups := int(abi.PaddedPieceSize(2048).Unpadded()) + + commP := func(b []byte) cid.Cid { + pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b))) + require.NoError(t, err) + + c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b))) + require.NoError(t, err) + + require.NoError(t, werr()) + + return c + } + + testCommEq := func(name string, in [][]byte, expect [][]byte) { + t.Run(name, func(t *testing.T) { + upi := make([]abi.PieceInfo, len(in)) + for i, b := range in { + upi[i] = abi.PieceInfo{ + Size: abi.UnpaddedPieceSize(len(b)).Padded(), + PieceCID: commP(b), + } + } + + sectorPi := []abi.PieceInfo{ + { + Size: 2048, + PieceCID: commP(bytes.Join(expect, nil)), + }, + } + + expectCid, err := GenerateUnsealedCID(pt, sectorPi) + require.NoError(t, err) + + actualCid, err := GenerateUnsealedCID(pt, upi) + require.NoError(t, err) + + require.Equal(t, expectCid, actualCid) + }) + } + + barr := func(b byte, den int) []byte { + return bytes.Repeat([]byte{b}, ups/den) + } + + // 0000 + testCommEq("zero", + nil, + [][]byte{barr(0, 1)}, + ) + + // 1111 + testCommEq("one", + [][]byte{barr(1, 1)}, + [][]byte{barr(1, 1)}, + ) + + // 11 00 + testCommEq("one|2", + [][]byte{barr(1, 2)}, + [][]byte{barr(1, 2), barr(0, 2)}, + ) + + // 1 0 00 + testCommEq("one|4", + [][]byte{barr(1, 4)}, + [][]byte{barr(1, 4), barr(0, 4), barr(0, 2)}, + ) + + // 11 2 0 + testCommEq("one|2-two|4", + [][]byte{barr(1, 2), barr(2, 4)}, + [][]byte{barr(1, 2), barr(2, 4), barr(0, 4)}, + ) + + // 1 0 22 + testCommEq("one|4-two|2", + [][]byte{barr(1, 4), barr(2, 2)}, + [][]byte{barr(1, 4), barr(0, 4), barr(2, 2)}, + ) + + // 1 0 22 0000 + testCommEq("one|8-two|4", + [][]byte{barr(1, 8), barr(2, 4)}, + [][]byte{barr(1, 8), barr(0, 8), barr(2, 4), barr(0, 2)}, + ) + + // 11 2 0 0000 + testCommEq("one|4-two|8", + [][]byte{barr(1, 4), barr(2, 8)}, + [][]byte{barr(1, 4), barr(2, 8), barr(0, 8), barr(0, 2)}, + ) + + // 1 0 22 3 0 00 4444 5 0 00 + testCommEq("one|16-two|8-three|16-four|4-five|16", + [][]byte{barr(1, 16), barr(2, 8), barr(3, 16), barr(4, 4), barr(5, 16)}, + [][]byte{barr(1, 16), barr(0, 16), barr(2, 8), barr(3, 16), barr(0, 16), barr(0, 8), barr(4, 4), barr(5, 16), barr(0, 16), barr(0, 8)}, + ) +} diff --git a/extern/sector-storage/ffiwrapper/types.go b/extern/sector-storage/ffiwrapper/types.go new file mode 100644 index 000000000..bc3c44f54 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/types.go @@ -0,0 +1,49 @@ +package ffiwrapper + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/ffiwrapper/basicfs" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +type Validator interface { + CanCommit(sector stores.SectorPaths) (bool, error) + CanProve(sector stores.SectorPaths) (bool, error) +} + +type StorageSealer interface { + storage.Sealer + storage.Storage +} + +type Storage interface { + storage.Prover + StorageSealer + + UnsealPiece(ctx context.Context, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error + ReadPiece(ctx context.Context, writer io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) +} + +type Verifier interface { + VerifySeal(abi.SealVerifyInfo) (bool, error) + VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) + VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) + + GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error) +} + +type SectorProvider interface { + // * returns storiface.ErrSectorNotFound if a requested existing sector doesn't exist + // * returns an error when allocate is set, and existing isn't, and the sector exists + AcquireSector(ctx context.Context, id abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, ptype stores.PathType) (stores.SectorPaths, func(), error) +} + +var _ SectorProvider = &basicfs.Provider{} diff --git a/extern/sector-storage/ffiwrapper/unseal_ranges.go b/extern/sector-storage/ffiwrapper/unseal_ranges.go new file mode 100644 index 000000000..0bc7b52df --- /dev/null +++ b/extern/sector-storage/ffiwrapper/unseal_ranges.go @@ -0,0 +1,26 @@ +package ffiwrapper + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-bitfield/rle" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/storiface" +) + +// merge gaps between ranges which are close to each other +// TODO: more benchmarking to come up with more optimal number +const mergeGaps = 32 << 20 + +// TODO const expandRuns = 16 << 20 // unseal more than requested for future requests + +func computeUnsealRanges(unsealed rlepluslazy.RunIterator, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (rlepluslazy.RunIterator, error) { + todo := pieceRun(offset.Padded(), size.Padded()) + todo, err := rlepluslazy.Subtract(todo, unsealed) + if err != nil { + return nil, xerrors.Errorf("compute todo-unsealed: %w", err) + } + + return rlepluslazy.JoinClose(todo, mergeGaps) +} diff --git a/extern/sector-storage/ffiwrapper/verifier_cgo.go b/extern/sector-storage/ffiwrapper/verifier_cgo.go new file mode 100644 index 000000000..1fecf9598 --- /dev/null +++ b/extern/sector-storage/ffiwrapper/verifier_cgo.go @@ -0,0 +1,119 @@ +//+build cgo + +package ffiwrapper + +import ( + "context" + "golang.org/x/xerrors" + + "go.opencensus.io/trace" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/stores" +) + +func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, error) { + randomness[31] &= 0x3f + privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWinningPoStProof) // TODO: FAULTS? + if err != nil { + return nil, err + } + defer done() + if len(skipped) > 0 { + return nil, xerrors.Errorf("pubSectorToPriv skipped sectors: %+v", skipped) + } + + return ffi.GenerateWinningPoSt(minerID, privsectors, randomness) +} + +func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, []abi.SectorID, error) { + randomness[31] &= 0x3f + privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) + if err != nil { + return nil, nil, xerrors.Errorf("gathering sector info: %w", err) + } + defer done() + + proof, err := ffi.GenerateWindowPoSt(minerID, privsectors, randomness) + return proof, skipped, err +} + +func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, faults []abi.SectorNumber, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error)) (ffi.SortedPrivateSectorInfo, []abi.SectorID, func(), error) { + fmap := map[abi.SectorNumber]struct{}{} + for _, fault := range faults { + fmap[fault] = struct{}{} + } + + var doneFuncs []func() + done := func() { + for _, df := range doneFuncs { + df() + } + } + + var skipped []abi.SectorID + var out []ffi.PrivateSectorInfo + for _, s := range sectorInfo { + if _, faulty := fmap[s.SectorNumber]; faulty { + continue + } + + sid := abi.SectorID{Miner: mid, Number: s.SectorNumber} + + paths, d, err := sb.sectors.AcquireSector(ctx, sid, stores.FTCache|stores.FTSealed, 0, stores.PathStorage) + if err != nil { + log.Warnw("failed to acquire sector, skipping", "sector", sid, "error", err) + skipped = append(skipped, sid) + continue + } + doneFuncs = append(doneFuncs, d) + + postProofType, err := rpt(s.SealProof) + if err != nil { + done() + return ffi.SortedPrivateSectorInfo{}, nil, nil, xerrors.Errorf("acquiring registered PoSt proof from sector info %+v: %w", s, err) + } + + out = append(out, ffi.PrivateSectorInfo{ + CacheDirPath: paths.Cache, + PoStProofType: postProofType, + SealedSectorPath: paths.Sealed, + SectorInfo: s, + }) + } + + return ffi.NewSortedPrivateSectorInfo(out...), skipped, done, nil +} + +var _ Verifier = ProofVerifier + +type proofVerifier struct{} + +var ProofVerifier = proofVerifier{} + +func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) { + return ffi.VerifySeal(info) +} + +func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) { + info.Randomness[31] &= 0x3f + _, span := trace.StartSpan(ctx, "VerifyWinningPoSt") + defer span.End() + + return ffi.VerifyWinningPoSt(info) +} + +func (proofVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) { + info.Randomness[31] &= 0x3f + _, span := trace.StartSpan(ctx, "VerifyWindowPoSt") + defer span.End() + + return ffi.VerifyWindowPoSt(info) +} + +func (proofVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, eligibleSectorCount uint64) ([]uint64, error) { + randomness[31] &= 0x3f + return ffi.GenerateWinningPoStSectorChallenge(proofType, minerID, randomness, eligibleSectorCount) +} diff --git a/extern/sector-storage/fr32/fr32.go b/extern/sector-storage/fr32/fr32.go new file mode 100644 index 000000000..fdf9d9223 --- /dev/null +++ b/extern/sector-storage/fr32/fr32.go @@ -0,0 +1,157 @@ +package fr32 + +import ( + "math/bits" + "runtime" + "sync" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +var MTTresh = uint64(32 << 20) + +func mtChunkCount(usz abi.PaddedPieceSize) uint64 { + threads := (uint64(usz)) / MTTresh + if threads > uint64(runtime.NumCPU()) { + threads = 1 << (bits.Len32(uint32(runtime.NumCPU()))) + } + if threads == 0 { + return 1 + } + if threads > 32 { + return 32 // avoid too large buffers + } + return threads +} + +func mt(in, out []byte, padLen int, op func(unpadded, padded []byte)) { + threads := mtChunkCount(abi.PaddedPieceSize(padLen)) + threadBytes := abi.PaddedPieceSize(padLen / int(threads)) + + var wg sync.WaitGroup + wg.Add(int(threads)) + + for i := 0; i < int(threads); i++ { + go func(thread int) { + defer wg.Done() + + start := threadBytes * abi.PaddedPieceSize(thread) + end := start + threadBytes + + op(in[start.Unpadded():end.Unpadded()], out[start:end]) + }(i) + } + wg.Wait() +} + +// Assumes len(in)%127==0 and len(out)%128==0 +func Pad(in, out []byte) { + if len(out) > int(MTTresh) { + mt(in, out, len(out), pad) + return + } + + pad(in, out) +} + +func pad(in, out []byte) { + chunks := len(out) / 128 + for chunk := 0; chunk < chunks; chunk++ { + inOff := chunk * 127 + outOff := chunk * 128 + + copy(out[outOff:outOff+31], in[inOff:inOff+31]) + + t := in[inOff+31] >> 6 + out[outOff+31] = in[inOff+31] & 0x3f + var v byte + + for i := 32; i < 64; i++ { + v = in[inOff+i] + out[outOff+i] = (v << 2) | t + t = v >> 6 + } + + t = v >> 4 + out[outOff+63] &= 0x3f + + for i := 64; i < 96; i++ { + v = in[inOff+i] + out[outOff+i] = (v << 4) | t + t = v >> 4 + } + + t = v >> 2 + out[outOff+95] &= 0x3f + + for i := 96; i < 127; i++ { + v = in[inOff+i] + out[outOff+i] = (v << 6) | t + t = v >> 2 + } + + out[outOff+127] = t & 0x3f + } +} + +// Assumes len(in)%128==0 and len(out)%127==0 +func Unpad(in []byte, out []byte) { + if len(in) > int(MTTresh) { + mt(out, in, len(in), unpad) + return + } + + unpad(out, in) +} + +func unpad(out, in []byte) { + chunks := len(in) / 128 + for chunk := 0; chunk < chunks; chunk++ { + inOffNext := chunk*128 + 1 + outOff := chunk * 127 + + at := in[chunk*128] + + for i := 0; i < 32; i++ { + next := in[i+inOffNext] + + out[outOff+i] = at + //out[i] |= next << 8 + + at = next + } + + out[outOff+31] |= at << 6 + + for i := 32; i < 64; i++ { + next := in[i+inOffNext] + + out[outOff+i] = at >> 2 + out[outOff+i] |= next << 6 + + at = next + } + + out[outOff+63] ^= (at << 6) ^ (at << 4) + + for i := 64; i < 96; i++ { + next := in[i+inOffNext] + + out[outOff+i] = at >> 4 + out[outOff+i] |= next << 4 + + at = next + } + + out[outOff+95] ^= (at << 4) ^ (at << 2) + + for i := 96; i < 127; i++ { + next := in[i+inOffNext] + + out[outOff+i] = at >> 6 + out[outOff+i] |= next << 2 + + at = next + } + } +} diff --git a/extern/sector-storage/fr32/fr32_ffi_cmp_test.go b/extern/sector-storage/fr32/fr32_ffi_cmp_test.go new file mode 100644 index 000000000..d9c3ba283 --- /dev/null +++ b/extern/sector-storage/fr32/fr32_ffi_cmp_test.go @@ -0,0 +1,66 @@ +package fr32_test + +import ( + "bytes" + "github.com/filecoin-project/sector-storage/fr32" + "io" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/require" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/ffiwrapper" +) + +func TestWriteTwoPcs(t *testing.T) { + tf, _ := ioutil.TempFile("/tmp/", "scrb-") + + paddedSize := abi.PaddedPieceSize(16 << 20) + n := 2 + + var rawBytes []byte + + for i := 0; i < n; i++ { + buf := bytes.Repeat([]byte{0xab * byte(i)}, int(paddedSize.Unpadded())) + rawBytes = append(rawBytes, buf...) + + rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) + + _, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil) + if err != nil { + panic(err) + } + if err := w(); err != nil { + panic(err) + } + } + + if _, err := tf.Seek(io.SeekStart, 0); err != nil { + panic(err) + } + + ffiBytes, err := ioutil.ReadAll(tf) + if err != nil { + panic(err) + } + + if err := tf.Close(); err != nil { + panic(err) + } + + if err := os.Remove(tf.Name()); err != nil { + panic(err) + } + + outBytes := make([]byte, int(paddedSize)*n) + fr32.Pad(rawBytes, outBytes) + require.Equal(t, ffiBytes, outBytes) + + unpadBytes := make([]byte, int(paddedSize.Unpadded())*n) + fr32.Unpad(ffiBytes, unpadBytes) + require.Equal(t, rawBytes, unpadBytes) +} diff --git a/extern/sector-storage/fr32/fr32_test.go b/extern/sector-storage/fr32/fr32_test.go new file mode 100644 index 000000000..219f10f5c --- /dev/null +++ b/extern/sector-storage/fr32/fr32_test.go @@ -0,0 +1,250 @@ +package fr32_test + +import ( + "bytes" + "io" + "io/ioutil" + "math/rand" + "os" + "testing" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/fr32" +) + +func padFFI(buf []byte) []byte { + rf, w, _ := ffiwrapper.ToReadableFile(bytes.NewReader(buf), int64(len(buf))) + tf, _ := ioutil.TempFile("/tmp/", "scrb-") + + _, _, _, err := ffi.WriteWithAlignment(abi.RegisteredSealProof_StackedDrg32GiBV1, rf, abi.UnpaddedPieceSize(len(buf)), tf, nil) + if err != nil { + panic(err) + } + if err := w(); err != nil { + panic(err) + } + + if _, err := tf.Seek(io.SeekStart, 0); err != nil { + panic(err) + } + + padded, err := ioutil.ReadAll(tf) + if err != nil { + panic(err) + } + + if err := tf.Close(); err != nil { + panic(err) + } + + if err := os.Remove(tf.Name()); err != nil { + panic(err) + } + + return padded +} + +func TestPadChunkFFI(t *testing.T) { + testByteChunk := func(b byte) func(*testing.T) { + return func(t *testing.T) { + var buf [128]byte + copy(buf[:], bytes.Repeat([]byte{b}, 127)) + + fr32.Pad(buf[:], buf[:]) + + expect := padFFI(bytes.Repeat([]byte{b}, 127)) + + require.Equal(t, expect, buf[:]) + } + } + + t.Run("ones", testByteChunk(0xff)) + t.Run("lsb1", testByteChunk(0x01)) + t.Run("msb1", testByteChunk(0x80)) + t.Run("zero", testByteChunk(0x0)) + t.Run("mid", testByteChunk(0x3c)) +} + +func TestPadChunkRandEqFFI(t *testing.T) { + for i := 0; i < 200; i++ { + var input [127]byte + rand.Read(input[:]) + + var buf [128]byte + + fr32.Pad(input[:], buf[:]) + + expect := padFFI(input[:]) + + require.Equal(t, expect, buf[:]) + } +} + +func TestRoundtrip(t *testing.T) { + testByteChunk := func(b byte) func(*testing.T) { + return func(t *testing.T) { + var buf [128]byte + input := bytes.Repeat([]byte{0x01}, 127) + + fr32.Pad(input, buf[:]) + + var out [127]byte + fr32.Unpad(buf[:], out[:]) + + require.Equal(t, input, out[:]) + } + } + + t.Run("ones", testByteChunk(0xff)) + t.Run("lsb1", testByteChunk(0x01)) + t.Run("msb1", testByteChunk(0x80)) + t.Run("zero", testByteChunk(0x0)) + t.Run("mid", testByteChunk(0x3c)) +} + +func TestRoundtripChunkRand(t *testing.T) { + for i := 0; i < 200; i++ { + var input [127]byte + rand.Read(input[:]) + + var buf [128]byte + copy(buf[:], input[:]) + + fr32.Pad(buf[:], buf[:]) + + var out [127]byte + fr32.Unpad(buf[:], out[:]) + + require.Equal(t, input[:], out[:]) + } +} + +func TestRoundtrip16MRand(t *testing.T) { + up := abi.PaddedPieceSize(16 << 20).Unpadded() + + input := make([]byte, up) + rand.Read(input[:]) + + buf := make([]byte, 16<<20) + + fr32.Pad(input, buf) + + out := make([]byte, up) + fr32.Unpad(buf, out) + + require.Equal(t, input, out) + + ffi := padFFI(input) + require.Equal(t, ffi, buf) +} + +func BenchmarkPadChunk(b *testing.B) { + var buf [128]byte + in := bytes.Repeat([]byte{0xff}, 127) + + b.SetBytes(127) + + for i := 0; i < b.N; i++ { + fr32.Pad(in, buf[:]) + } +} + +func BenchmarkChunkRoundtrip(b *testing.B) { + var buf [128]byte + copy(buf[:], bytes.Repeat([]byte{0xff}, 127)) + var out [127]byte + + b.SetBytes(127) + + for i := 0; i < b.N; i++ { + fr32.Pad(buf[:], buf[:]) + fr32.Unpad(buf[:], out[:]) + } +} + +func BenchmarkUnpadChunk(b *testing.B) { + var buf [128]byte + copy(buf[:], bytes.Repeat([]byte{0xff}, 127)) + + fr32.Pad(buf[:], buf[:]) + var out [127]byte + + b.SetBytes(127) + b.ReportAllocs() + + bs := buf[:] + + for i := 0; i < b.N; i++ { + fr32.Unpad(bs, out[:]) + } +} + +func BenchmarkUnpad16MChunk(b *testing.B) { + up := abi.PaddedPieceSize(16 << 20).Unpadded() + + var buf [16 << 20]byte + + fr32.Pad(bytes.Repeat([]byte{0xff}, int(up)), buf[:]) + var out [16 << 20]byte + + b.SetBytes(16 << 20) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + fr32.Unpad(buf[:], out[:]) + } +} + +func BenchmarkPad16MChunk(b *testing.B) { + up := abi.PaddedPieceSize(16 << 20).Unpadded() + + var buf [16 << 20]byte + + in := bytes.Repeat([]byte{0xff}, int(up)) + + b.SetBytes(16 << 20) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + fr32.Pad(in, buf[:]) + } +} + +func BenchmarkPad1GChunk(b *testing.B) { + up := abi.PaddedPieceSize(1 << 30).Unpadded() + + var buf [1 << 30]byte + + in := bytes.Repeat([]byte{0xff}, int(up)) + + b.SetBytes(1 << 30) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + fr32.Pad(in, buf[:]) + } +} + +func BenchmarkUnpad1GChunk(b *testing.B) { + up := abi.PaddedPieceSize(1 << 30).Unpadded() + + var buf [1 << 30]byte + + fr32.Pad(bytes.Repeat([]byte{0xff}, int(up)), buf[:]) + var out [1 << 30]byte + + b.SetBytes(1 << 30) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + fr32.Unpad(buf[:], out[:]) + } +} diff --git a/extern/sector-storage/fr32/readers.go b/extern/sector-storage/fr32/readers.go new file mode 100644 index 000000000..8a1bbe087 --- /dev/null +++ b/extern/sector-storage/fr32/readers.go @@ -0,0 +1,133 @@ +package fr32 + +import ( + "io" + "math/bits" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type unpadReader struct { + src io.Reader + + left uint64 + work []byte +} + +func NewUnpadReader(src io.Reader, sz abi.PaddedPieceSize) (io.Reader, error) { + if err := sz.Validate(); err != nil { + return nil, xerrors.Errorf("bad piece size: %w", err) + } + + buf := make([]byte, MTTresh*mtChunkCount(sz)) + + return &unpadReader{ + src: src, + + left: uint64(sz), + work: buf, + }, nil +} + +func (r *unpadReader) Read(out []byte) (int, error) { + if r.left == 0 { + return 0, io.EOF + } + + chunks := len(out) / 127 + + outTwoPow := 1 << (63 - bits.LeadingZeros64(uint64(chunks*128))) + + if err := abi.PaddedPieceSize(outTwoPow).Validate(); err != nil { + return 0, xerrors.Errorf("output must be of valid padded piece size: %w", err) + } + + todo := abi.PaddedPieceSize(outTwoPow) + if r.left < uint64(todo) { + todo = abi.PaddedPieceSize(1 << (63 - bits.LeadingZeros64(r.left))) + } + + r.left -= uint64(todo) + + n, err := r.src.Read(r.work[:todo]) + if err != nil && err != io.EOF { + return n, err + } + + if n != int(todo) { + return 0, xerrors.Errorf("didn't read enough: %w", err) + } + + Unpad(r.work[:todo], out[:todo.Unpadded()]) + + return int(todo.Unpadded()), err +} + +type padWriter struct { + dst io.Writer + + stash []byte + work []byte +} + +func NewPadWriter(dst io.Writer) io.WriteCloser { + return &padWriter{ + dst: dst, + } +} + +func (w *padWriter) Write(p []byte) (int, error) { + in := p + + if len(p)+len(w.stash) < 127 { + w.stash = append(w.stash, p...) + return len(p), nil + } + + if len(w.stash) != 0 { + in = append(w.stash, in...) + } + + for { + pieces := subPieces(abi.UnpaddedPieceSize(len(in))) + biggest := pieces[len(pieces)-1] + + if abi.PaddedPieceSize(cap(w.work)) < biggest.Padded() { + w.work = make([]byte, 0, biggest.Padded()) + } + + Pad(in[:int(biggest)], w.work[:int(biggest.Padded())]) + + n, err := w.dst.Write(w.work[:int(biggest.Padded())]) + if err != nil { + return int(abi.PaddedPieceSize(n).Unpadded()), err + } + + in = in[biggest:] + + if len(in) < 127 { + if cap(w.stash) < len(in) { + w.stash = make([]byte, 0, len(in)) + } + w.stash = w.stash[:len(in)] + copy(w.stash, in) + + return len(p), nil + } + } +} + +func (w *padWriter) Close() error { + if len(w.stash) > 0 { + return xerrors.Errorf("still have %d unprocessed bytes", len(w.stash)) + } + + // allow gc + w.stash = nil + w.work = nil + w.dst = nil + + return nil +} diff --git a/extern/sector-storage/fr32/readers_test.go b/extern/sector-storage/fr32/readers_test.go new file mode 100644 index 000000000..f0f1e21bc --- /dev/null +++ b/extern/sector-storage/fr32/readers_test.go @@ -0,0 +1,34 @@ +package fr32_test + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/fr32" +) + +func TestUnpadReader(t *testing.T) { + ps := abi.PaddedPieceSize(64 << 20).Unpadded() + + raw := bytes.Repeat([]byte{0x77}, int(ps)) + + padOut := make([]byte, ps.Padded()) + fr32.Pad(raw, padOut) + + r, err := fr32.NewUnpadReader(bytes.NewReader(padOut), ps.Padded()) + if err != nil { + t.Fatal(err) + } + + readered, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, raw, readered) +} diff --git a/extern/sector-storage/fr32/utils.go b/extern/sector-storage/fr32/utils.go new file mode 100644 index 000000000..9f4093c40 --- /dev/null +++ b/extern/sector-storage/fr32/utils.go @@ -0,0 +1,31 @@ +package fr32 + +import ( + "math/bits" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +func subPieces(in abi.UnpaddedPieceSize) []abi.UnpaddedPieceSize { + // Convert to in-sector bytes for easier math: + // + // (we convert to sector bytes as they are nice round binary numbers) + + w := uint64(in.Padded()) + + out := make([]abi.UnpaddedPieceSize, bits.OnesCount64(w)) + for i := range out { + // Extract the next lowest non-zero bit + next := bits.TrailingZeros64(w) + psize := uint64(1) << next + // e.g: if the number is 0b010100, psize will be 0b000100 + + // set that bit to 0 by XORing it, so the next iteration looks at the + // next bit + w ^= psize + + // Add the piece size to the list of pieces we need to create + out[i] = abi.PaddedPieceSize(psize).Unpadded() + } + return out +} diff --git a/extern/sector-storage/fsutil/dealloc_linux.go b/extern/sector-storage/fsutil/dealloc_linux.go new file mode 100644 index 000000000..0b20c568d --- /dev/null +++ b/extern/sector-storage/fsutil/dealloc_linux.go @@ -0,0 +1,28 @@ +package fsutil + +import ( + "os" + "syscall" + + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("fsutil") + +const FallocFlPunchHole = 0x02 // linux/falloc.h + +func Deallocate(file *os.File, offset int64, length int64) error { + if length == 0 { + return nil + } + + err := syscall.Fallocate(int(file.Fd()), FallocFlPunchHole, offset, length) + if errno, ok := err.(syscall.Errno); ok { + if errno == syscall.EOPNOTSUPP || errno == syscall.ENOSYS { + log.Warnf("could not deallocate space, ignoring: %v", errno) + err = nil // log and ignore + } + } + + return err +} diff --git a/extern/sector-storage/fsutil/dealloc_other.go b/extern/sector-storage/fsutil/dealloc_other.go new file mode 100644 index 000000000..4f8347951 --- /dev/null +++ b/extern/sector-storage/fsutil/dealloc_other.go @@ -0,0 +1,17 @@ +// +build !linux + +package fsutil + +import ( + "os" + + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("fsutil") + +func Deallocate(file *os.File, offset int64, length int64) error { + log.Warnf("deallocating space not supported") + + return nil +} diff --git a/extern/sector-storage/fsutil/filesize_unix.go b/extern/sector-storage/fsutil/filesize_unix.go new file mode 100644 index 000000000..dacdcd515 --- /dev/null +++ b/extern/sector-storage/fsutil/filesize_unix.go @@ -0,0 +1,29 @@ +package fsutil + +import ( + "os" + "syscall" + + "golang.org/x/xerrors" +) + +type SizeInfo struct { + OnDisk int64 +} + +// FileSize returns bytes used by a file on disk +func FileSize(path string) (SizeInfo, error) { + var stat syscall.Stat_t + if err := syscall.Stat(path, &stat); err != nil { + if err == syscall.ENOENT { + return SizeInfo{}, os.ErrNotExist + } + return SizeInfo{}, xerrors.Errorf("stat: %w", err) + } + + // NOTE: stat.Blocks is in 512B blocks, NOT in stat.Blksize + // See https://www.gnu.org/software/libc/manual/html_node/Attribute-Meanings.html + return SizeInfo{ + int64(stat.Blocks) * 512, // NOTE: int64 cast is needed on osx + }, nil +} diff --git a/extern/sector-storage/fsutil/statfs.go b/extern/sector-storage/fsutil/statfs.go new file mode 100644 index 000000000..2a00ccb9a --- /dev/null +++ b/extern/sector-storage/fsutil/statfs.go @@ -0,0 +1,7 @@ +package fsutil + +type FsStat struct { + Capacity int64 + Available int64 // Available to use for sector storage + Reserved int64 +} diff --git a/extern/sector-storage/fsutil/statfs_unix.go b/extern/sector-storage/fsutil/statfs_unix.go new file mode 100644 index 000000000..7fcb8af37 --- /dev/null +++ b/extern/sector-storage/fsutil/statfs_unix.go @@ -0,0 +1,19 @@ +package fsutil + +import ( + "syscall" + + "golang.org/x/xerrors" +) + +func Statfs(path string) (FsStat, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return FsStat{}, xerrors.Errorf("statfs: %w", err) + } + + return FsStat{ + Capacity: int64(stat.Blocks) * int64(stat.Bsize), + Available: int64(stat.Bavail) * int64(stat.Bsize), + }, nil +} diff --git a/extern/sector-storage/fsutil/statfs_windows.go b/extern/sector-storage/fsutil/statfs_windows.go new file mode 100644 index 000000000..d78565182 --- /dev/null +++ b/extern/sector-storage/fsutil/statfs_windows.go @@ -0,0 +1,28 @@ +package fsutil + +import ( + "syscall" + "unsafe" +) + +func Statfs(volumePath string) (FsStat, error) { + // From https://github.com/ricochet2200/go-disk-usage/blob/master/du/diskusage_windows.go + + h := syscall.MustLoadDLL("kernel32.dll") + c := h.MustFindProc("GetDiskFreeSpaceExW") + + var freeBytes int64 + var totalBytes int64 + var availBytes int64 + + c.Call( + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(volumePath))), + uintptr(unsafe.Pointer(&freeBytes)), + uintptr(unsafe.Pointer(&totalBytes)), + uintptr(unsafe.Pointer(&availBytes))) + + return FsStat{ + Capacity: totalBytes, + Available: availBytes, + }, nil +} diff --git a/extern/sector-storage/go.mod b/extern/sector-storage/go.mod new file mode 100644 index 000000000..a6fb3ee89 --- /dev/null +++ b/extern/sector-storage/go.mod @@ -0,0 +1,32 @@ +module github.com/filecoin-project/sector-storage + +go 1.13 + +require ( + github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e + github.com/elastic/go-sysinfo v1.3.0 + github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d + github.com/filecoin-project/go-bitfield v0.1.2 + github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f + github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 + github.com/filecoin-project/specs-actors v0.8.2 + github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea + github.com/google/uuid v1.1.1 + github.com/gorilla/mux v1.7.4 + github.com/hashicorp/go-multierror v1.0.0 + github.com/ipfs/go-cid v0.0.6 + github.com/ipfs/go-ipfs-files v0.0.7 + github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 // indirect + github.com/ipfs/go-log v1.0.4 + github.com/ipfs/go-log/v2 v2.0.5 + github.com/mattn/go-isatty v0.0.9 // indirect + github.com/mitchellh/go-homedir v1.1.0 + github.com/stretchr/testify v1.6.1 + go.opencensus.io v0.22.3 + golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 // indirect + golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d // indirect + golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + honnef.co/go/tools v0.0.1-2020.1.3 // indirect +) diff --git a/extern/sector-storage/go.sum b/extern/sector-storage/go.sum new file mode 100644 index 000000000..942ff725c --- /dev/null +++ b/extern/sector-storage/go.sum @@ -0,0 +1,364 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= +github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= +github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be h1:TooKBwR/g8jG0hZ3lqe9S5sy2vTUcLOZLlz3M5wGn2E= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= +github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.1.2 h1:TjLregCoyP1/5lm7WCM0axyV1myIHwbjGa21skuu5tk= +github.com/filecoin-project/go-bitfield v0.1.2/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.8.2 h1:fpAPOPqWqmzJCWHpm6P1XDRSpQrxyY5Pzh5H3doYs7Q= +github.com/filecoin-project/specs-actors v0.8.2/go.mod h1:Q3ACV5kBLvqPaYbthc/J1lGMJ5OwogmD9pzdtPRMdCw= +github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= +github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= +github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= +github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= +github.com/ipfs/go-ipfs-files v0.0.7 h1:s5BRD12ndahqYifeH1S8Z73zqZhR+3IdKYAG9PiETs0= +github.com/ipfs/go-ipfs-files v0.0.7/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= +github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTiSA= +github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 h1:jIVle1vGSzxyUhseYNEqd7qcDVRrIbJ7UxGwao70cF0= +github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= +github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= +github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.4 h1:6nLQdX4W8P9yZZFH7mO+X/PzjN8Laozm/lMJ6esdgzY= +github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= +github.com/ipfs/go-log/v2 v2.0.5 h1:fL4YI+1g5V/b1Yxr1qAiXTMg1H8z9vx/VmJxBuQMHvU= +github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= +github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= +github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0 h1:c8R11WC8m7KNMkTv/0+Be8vvwo4I3/Ut9AC2FW8fX3U= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= +github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e h1:JY8o/ebUUrCYetWmjRCNghxC59cOEaili83rxPRQCLw= +github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= +github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377 h1:LHFlP/ktDvOnCap7PsT87cs7Gwd0p+qv6Qm5g2ZPR+I= +github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww= +golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae h1:QoJmnb9uyPCrH8GIg9uRLn4Ta45yhcQtpymCd0AavO8= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d h1:62ap6LNOjDU6uGmKXHJbSfciMoV+FeI1sRXx/pLDL44= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 h1:OXjomkWHhzUx4+HldlJ2TsMxJdWgEo5CTtspD1wdhdk= +golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= diff --git a/extern/sector-storage/localworker.go b/extern/sector-storage/localworker.go new file mode 100644 index 000000000..14ed1cd0b --- /dev/null +++ b/extern/sector-storage/localworker.go @@ -0,0 +1,298 @@ +package sectorstorage + +import ( + "context" + "io" + "os" + "runtime" + + "github.com/elastic/go-sysinfo" + "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/specs-actors/actors/abi" + storage2 "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +var pathTypes = []stores.SectorFileType{stores.FTUnsealed, stores.FTSealed, stores.FTCache} + +type WorkerConfig struct { + SealProof abi.RegisteredSealProof + TaskTypes []sealtasks.TaskType +} + +type LocalWorker struct { + scfg *ffiwrapper.Config + storage stores.Store + localStore *stores.Local + sindex stores.SectorIndex + + acceptTasks map[sealtasks.TaskType]struct{} +} + +func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex) *LocalWorker { + acceptTasks := map[sealtasks.TaskType]struct{}{} + for _, taskType := range wcfg.TaskTypes { + acceptTasks[taskType] = struct{}{} + } + + return &LocalWorker{ + scfg: &ffiwrapper.Config{ + SealProofType: wcfg.SealProof, + }, + storage: store, + localStore: local, + sindex: sindex, + + acceptTasks: acceptTasks, + } +} + +type localWorkerPathProvider struct { + w *LocalWorker + op stores.AcquireMode +} + +func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing stores.PathType) (stores.SectorPaths, func(), error) { + + paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, l.w.scfg.SealProofType, existing, allocate, sealing, l.op) + if err != nil { + return stores.SectorPaths{}, nil, err + } + + releaseStorage, err := l.w.localStore.Reserve(ctx, sector, l.w.scfg.SealProofType, allocate, storageIDs, stores.FSOverheadSeal) + if err != nil { + return stores.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err) + } + + log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths) + + return paths, func() { + releaseStorage() + + for _, fileType := range pathTypes { + if fileType&allocate == 0 { + continue + } + + sid := stores.PathByType(storageIDs, fileType) + + if err := l.w.sindex.StorageDeclareSector(ctx, stores.ID(sid), sector, fileType, l.op == stores.AcquireMove); err != nil { + log.Errorf("declare sector error: %+v", err) + } + } + }, nil +} + +func (l *LocalWorker) sb() (ffiwrapper.Storage, error) { + return ffiwrapper.New(&localWorkerPathProvider{w: l}, l.scfg) +} + +func (l *LocalWorker) NewSector(ctx context.Context, sector abi.SectorID) error { + sb, err := l.sb() + if err != nil { + return err + } + + return sb.NewSector(ctx, sector) +} + +func (l *LocalWorker) AddPiece(ctx context.Context, sector abi.SectorID, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { + sb, err := l.sb() + if err != nil { + return abi.PieceInfo{}, err + } + + return sb.AddPiece(ctx, sector, epcs, sz, r) +} + +func (l *LocalWorker) Fetch(ctx context.Context, sector abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error { + _, done, err := (&localWorkerPathProvider{w: l, op: am}).AcquireSector(ctx, sector, fileType, stores.FTNone, ptype) + if err != nil { + return err + } + done() + return nil +} + +func (l *LocalWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { + { + // cleanup previous failed attempts if they exist + if err := l.storage.Remove(ctx, sector, stores.FTSealed, true); err != nil { + return nil, xerrors.Errorf("cleaning up sealed data: %w", err) + } + + if err := l.storage.Remove(ctx, sector, stores.FTCache, true); err != nil { + return nil, xerrors.Errorf("cleaning up cache data: %w", err) + } + } + + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealPreCommit1(ctx, sector, ticket, pieces) +} + +func (l *LocalWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { + sb, err := l.sb() + if err != nil { + return storage2.SectorCids{}, err + } + + return sb.SealPreCommit2(ctx, sector, phase1Out) +} + +func (l *LocalWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit1(ctx, sector, ticket, seed, pieces, cids) +} + +func (l *LocalWorker) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit2(ctx, sector, phase1Out) +} + +func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage2.Range) error { + sb, err := l.sb() + if err != nil { + return err + } + + if err := sb.FinalizeSector(ctx, sector, keepUnsealed); err != nil { + return xerrors.Errorf("finalizing sector: %w", err) + } + + if len(keepUnsealed) == 0 { + if err := l.storage.Remove(ctx, sector, stores.FTUnsealed, true); err != nil { + return xerrors.Errorf("removing unsealed data: %w", err) + } + } + + return nil +} + +func (l *LocalWorker) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage2.Range) error { + return xerrors.Errorf("implement me") +} + +func (l *LocalWorker) Remove(ctx context.Context, sector abi.SectorID) error { + var err error + + if rerr := l.storage.Remove(ctx, sector, stores.FTSealed, true); rerr != nil { + err = multierror.Append(err, xerrors.Errorf("removing sector (sealed): %w", rerr)) + } + if rerr := l.storage.Remove(ctx, sector, stores.FTCache, true); rerr != nil { + err = multierror.Append(err, xerrors.Errorf("removing sector (cache): %w", rerr)) + } + if rerr := l.storage.Remove(ctx, sector, stores.FTUnsealed, true); rerr != nil { + err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr)) + } + + return err +} + +func (l *LocalWorker) MoveStorage(ctx context.Context, sector abi.SectorID) error { + if err := l.storage.MoveStorage(ctx, sector, l.scfg.SealProofType, stores.FTSealed|stores.FTCache); err != nil { + return xerrors.Errorf("moving sealed data to storage: %w", err) + } + + return nil +} + +func (l *LocalWorker) UnsealPiece(ctx context.Context, sector abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) error { + sb, err := l.sb() + if err != nil { + return err + } + + if err := sb.UnsealPiece(ctx, sector, index, size, randomness, cid); err != nil { + return xerrors.Errorf("unsealing sector: %w", err) + } + + if err := l.storage.RemoveCopies(ctx, sector, stores.FTSealed); err != nil { + return xerrors.Errorf("removing source data: %w", err) + } + + if err := l.storage.RemoveCopies(ctx, sector, stores.FTCache); err != nil { + return xerrors.Errorf("removing source data: %w", err) + } + + return nil +} + +func (l *LocalWorker) ReadPiece(ctx context.Context, writer io.Writer, sector abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + sb, err := l.sb() + if err != nil { + return false, err + } + + return sb.ReadPiece(ctx, writer, sector, index, size) +} + +func (l *LocalWorker) TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) { + return l.acceptTasks, nil +} + +func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { + return l.localStore.Local(ctx) +} + +func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) { + hostname, err := os.Hostname() // TODO: allow overriding from config + if err != nil { + panic(err) + } + + gpus, err := ffi.GetGPUDevices() + if err != nil { + log.Errorf("getting gpu devices failed: %+v", err) + } + + h, err := sysinfo.Host() + if err != nil { + return storiface.WorkerInfo{}, xerrors.Errorf("getting host info: %w", err) + } + + mem, err := h.Memory() + if err != nil { + return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) + } + + return storiface.WorkerInfo{ + Hostname: hostname, + Resources: storiface.WorkerResources{ + MemPhysical: mem.Total, + MemSwap: mem.VirtualTotal, + MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process + CPUs: uint64(runtime.NumCPU()), + GPUs: gpus, + }, + }, nil +} + +func (l *LocalWorker) Closing(ctx context.Context) (<-chan struct{}, error) { + return make(chan struct{}), nil +} + +func (l *LocalWorker) Close() error { + return nil +} + +var _ Worker = &LocalWorker{} diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go new file mode 100644 index 000000000..e9fa1ccd4 --- /dev/null +++ b/extern/sector-storage/manager.go @@ -0,0 +1,508 @@ +package sectorstorage + +import ( + "context" + "errors" + "github.com/filecoin-project/sector-storage/fsutil" + "io" + "net/http" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +var log = logging.Logger("advmgr") + +var ErrNoWorkers = errors.New("no suitable workers found") + +type URLs []string + +type Worker interface { + ffiwrapper.StorageSealer + + MoveStorage(ctx context.Context, sector abi.SectorID) error + + Fetch(ctx context.Context, s abi.SectorID, ft stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error + UnsealPiece(context.Context, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error + ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize) (bool, error) + + TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) + + // Returns paths accessible to the worker + Paths(context.Context) ([]stores.StoragePath, error) + + Info(context.Context) (storiface.WorkerInfo, error) + + // returns channel signalling worker shutdown + Closing(context.Context) (<-chan struct{}, error) + + Close() error +} + +type SectorManager interface { + SectorSize() abi.SectorSize + + ReadPiece(context.Context, io.Writer, abi.SectorID, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) error + + ffiwrapper.StorageSealer + storage.Prover + FaultTracker +} + +type WorkerID uint64 + +type Manager struct { + scfg *ffiwrapper.Config + + ls stores.LocalStorage + storage *stores.Remote + localStore *stores.Local + remoteHnd *stores.FetchHandler + index stores.SectorIndex + + sched *scheduler + + storage.Prover +} + +type SealerConfig struct { + ParallelFetchLimit int + + // Local worker config + AllowPreCommit1 bool + AllowPreCommit2 bool + AllowCommit bool + AllowUnseal bool +} + +type StorageAuth http.Header + +func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg *ffiwrapper.Config, sc SealerConfig, urls URLs, sa StorageAuth) (*Manager, error) { + lstor, err := stores.NewLocal(ctx, ls, si, urls) + if err != nil { + return nil, err + } + + prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor, index: si}, cfg) + if err != nil { + return nil, xerrors.Errorf("creating prover instance: %w", err) + } + + stor := stores.NewRemote(lstor, si, http.Header(sa), sc.ParallelFetchLimit) + + m := &Manager{ + scfg: cfg, + + ls: ls, + storage: stor, + localStore: lstor, + remoteHnd: &stores.FetchHandler{Local: lstor}, + index: si, + + sched: newScheduler(cfg.SealProofType), + + Prover: prover, + } + + go m.sched.runSched() + + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTFetch, sealtasks.TTReadUnsealed, + } + if sc.AllowPreCommit1 { + localTasks = append(localTasks, sealtasks.TTPreCommit1) + } + if sc.AllowPreCommit2 { + localTasks = append(localTasks, sealtasks.TTPreCommit2) + } + if sc.AllowCommit { + localTasks = append(localTasks, sealtasks.TTCommit2) + } + if sc.AllowUnseal { + localTasks = append(localTasks, sealtasks.TTUnseal) + } + + err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ + SealProof: cfg.SealProofType, + TaskTypes: localTasks, + }, stor, lstor, si)) + if err != nil { + return nil, xerrors.Errorf("adding local worker: %w", err) + } + + return m, nil +} + +func (m *Manager) AddLocalStorage(ctx context.Context, path string) error { + path, err := homedir.Expand(path) + if err != nil { + return xerrors.Errorf("expanding local path: %w", err) + } + + if err := m.localStore.OpenPath(ctx, path); err != nil { + return xerrors.Errorf("opening local path: %w", err) + } + + if err := m.ls.SetStorage(func(sc *stores.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, stores.LocalPath{Path: path}) + }); err != nil { + return xerrors.Errorf("get storage config: %w", err) + } + return nil +} + +func (m *Manager) AddWorker(ctx context.Context, w Worker) error { + info, err := w.Info(ctx) + if err != nil { + return xerrors.Errorf("getting worker info: %w", err) + } + + m.sched.newWorkers <- &workerHandle{ + w: w, + wt: &workTracker{ + running: map[uint64]storiface.WorkerJob{}, + }, + info: info, + preparing: &activeResources{}, + active: &activeResources{}, + } + return nil +} + +func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + m.remoteHnd.ServeHTTP(w, r) +} + +func (m *Manager) SectorSize() abi.SectorSize { + sz, _ := m.scfg.SealProofType.SectorSize() + return sz +} + +func schedNop(context.Context, Worker) error { + return nil +} + +func schedFetch(sector abi.SectorID, ft stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) func(context.Context, Worker) error { + return func(ctx context.Context, worker Worker) error { + return worker.Fetch(ctx, sector, ft, ptype, am) + } +} + +func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTSealed|stores.FTCache, stores.FTUnsealed); err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + // passing 0 spt because we only need it when allowFetch is true + best, err := m.index.StorageFindSector(ctx, sector, stores.FTUnsealed, 0, false) + if err != nil { + return xerrors.Errorf("read piece: checking for already existing unsealed sector: %w", err) + } + + var selector WorkerSelector + if len(best) == 0 { // new + selector = newAllocSelector(m.index, stores.FTUnsealed, stores.PathSealing) + } else { // append to existing + selector = newExistingSelector(m.index, sector, stores.FTUnsealed, false) + } + + var readOk bool + + if len(best) > 0 { + // There is unsealed sector, see if we can read from it + + selector = newExistingSelector(m.index, sector, stores.FTUnsealed, false) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTReadUnsealed, selector, schedFetch(sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove), func(ctx context.Context, w Worker) error { + readOk, err = w.ReadPiece(ctx, sink, sector, offset, size) + return err + }) + if err != nil { + return xerrors.Errorf("reading piece from sealed sector: %w", err) + } + + if readOk { + return nil + } + } + + unsealFetch := func(ctx context.Context, worker Worker) error { + if err := worker.Fetch(ctx, sector, stores.FTSealed|stores.FTCache, stores.PathSealing, stores.AcquireCopy); err != nil { + return xerrors.Errorf("copy sealed/cache sector data: %w", err) + } + + if len(best) > 0 { + if err := worker.Fetch(ctx, sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove); err != nil { + return xerrors.Errorf("copy unsealed sector data: %w", err) + } + } + return nil + } + + err = m.sched.Schedule(ctx, sector, sealtasks.TTUnseal, selector, unsealFetch, func(ctx context.Context, w Worker) error { + return w.UnsealPiece(ctx, sector, offset, size, ticket, unsealed) + }) + if err != nil { + return err + } + + selector = newExistingSelector(m.index, sector, stores.FTUnsealed, false) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTReadUnsealed, selector, schedFetch(sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove), func(ctx context.Context, w Worker) error { + readOk, err = w.ReadPiece(ctx, sink, sector, offset, size) + return err + }) + if err != nil { + return xerrors.Errorf("reading piece from sealed sector: %w", err) + } + + if readOk { + return xerrors.Errorf("failed to read unsealed piece") + } + + return nil +} + +func (m *Manager) NewSector(ctx context.Context, sector abi.SectorID) error { + log.Warnf("stub NewSector") + return nil +} + +func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTNone, stores.FTUnsealed); err != nil { + return abi.PieceInfo{}, xerrors.Errorf("acquiring sector lock: %w", err) + } + + var selector WorkerSelector + var err error + if len(existingPieces) == 0 { // new + selector = newAllocSelector(m.index, stores.FTUnsealed, stores.PathSealing) + } else { // use existing + selector = newExistingSelector(m.index, sector, stores.FTUnsealed, false) + } + + var out abi.PieceInfo + err = m.sched.Schedule(ctx, sector, sealtasks.TTAddPiece, selector, schedNop, func(ctx context.Context, w Worker) error { + p, err := w.AddPiece(ctx, sector, existingPieces, sz, r) + if err != nil { + return err + } + out = p + return nil + }) + + return out, err +} + +func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache); err != nil { + return nil, xerrors.Errorf("acquiring sector lock: %w", err) + } + + // TODO: also consider where the unsealed data sits + + selector := newAllocSelector(m.index, stores.FTCache|stores.FTSealed, stores.PathSealing) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit1, selector, schedFetch(sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove), func(ctx context.Context, w Worker) error { + p, err := w.SealPreCommit1(ctx, sector, ticket, pieces) + if err != nil { + return err + } + out = p + return nil + }) + + return out, err +} + +func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (out storage.SectorCids, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTSealed, stores.FTCache); err != nil { + return storage.SectorCids{}, xerrors.Errorf("acquiring sector lock: %w", err) + } + + selector := newExistingSelector(m.index, sector, stores.FTCache|stores.FTSealed, true) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTPreCommit2, selector, schedFetch(sector, stores.FTCache|stores.FTSealed, stores.PathSealing, stores.AcquireMove), func(ctx context.Context, w Worker) error { + p, err := w.SealPreCommit2(ctx, sector, phase1Out) + if err != nil { + return err + } + out = p + return nil + }) + return out, err +} + +func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (out storage.Commit1Out, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTSealed, stores.FTCache); err != nil { + return storage.Commit1Out{}, xerrors.Errorf("acquiring sector lock: %w", err) + } + + // NOTE: We set allowFetch to false in so that we always execute on a worker + // with direct access to the data. We want to do that because this step is + // generally very cheap / fast, and transferring data is not worth the effort + selector := newExistingSelector(m.index, sector, stores.FTCache|stores.FTSealed, false) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTCommit1, selector, schedFetch(sector, stores.FTCache|stores.FTSealed, stores.PathSealing, stores.AcquireMove), func(ctx context.Context, w Worker) error { + p, err := w.SealCommit1(ctx, sector, ticket, seed, pieces, cids) + if err != nil { + return err + } + out = p + return nil + }) + return out, err +} + +func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (out storage.Proof, err error) { + selector := newTaskSelector() + + err = m.sched.Schedule(ctx, sector, sealtasks.TTCommit2, selector, schedNop, func(ctx context.Context, w Worker) error { + p, err := w.SealCommit2(ctx, sector, phase1Out) + if err != nil { + return err + } + out = p + return nil + }) + + return out, err +} + +func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTNone, stores.FTSealed|stores.FTUnsealed|stores.FTCache); err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + unsealed := stores.FTUnsealed + { + unsealedStores, err := m.index.StorageFindSector(ctx, sector, stores.FTUnsealed, 0, false) + if err != nil { + return xerrors.Errorf("finding unsealed sector: %w", err) + } + + if len(unsealedStores) == 0 { // Is some edge-cases unsealed sector may not exist already, that's fine + unsealed = stores.FTNone + } + } + + selector := newExistingSelector(m.index, sector, stores.FTCache|stores.FTSealed, false) + + err := m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector, + schedFetch(sector, stores.FTCache|stores.FTSealed|unsealed, stores.PathSealing, stores.AcquireMove), + func(ctx context.Context, w Worker) error { + return w.FinalizeSector(ctx, sector, keepUnsealed) + }) + if err != nil { + return err + } + + fetchSel := newAllocSelector(m.index, stores.FTCache|stores.FTSealed, stores.PathStorage) + moveUnsealed := unsealed + { + if len(keepUnsealed) == 0 { + moveUnsealed = stores.FTNone + } + } + + err = m.sched.Schedule(ctx, sector, sealtasks.TTFetch, fetchSel, + schedFetch(sector, stores.FTCache|stores.FTSealed|moveUnsealed, stores.PathStorage, stores.AcquireMove), + func(ctx context.Context, w Worker) error { + return w.MoveStorage(ctx, sector) + }) + if err != nil { + return xerrors.Errorf("moving sector to storage: %w", err) + } + + return nil +} + +func (m *Manager) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { + log.Warnw("ReleaseUnsealed todo") + return nil +} + +func (m *Manager) Remove(ctx context.Context, sector abi.SectorID) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector, stores.FTNone, stores.FTSealed|stores.FTUnsealed|stores.FTCache); err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + unsealed := stores.FTUnsealed + { + unsealedStores, err := m.index.StorageFindSector(ctx, sector, stores.FTUnsealed, 0, false) + if err != nil { + return xerrors.Errorf("finding unsealed sector: %w", err) + } + + if len(unsealedStores) == 0 { // can be already removed + unsealed = stores.FTNone + } + } + + selector := newExistingSelector(m.index, sector, stores.FTCache|stores.FTSealed, false) + + return m.sched.Schedule(ctx, sector, sealtasks.TTFinalize, selector, + schedFetch(sector, stores.FTCache|stores.FTSealed|unsealed, stores.PathStorage, stores.AcquireMove), + func(ctx context.Context, w Worker) error { + return w.Remove(ctx, sector) + }) +} + +func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { + l, err := m.localStore.Local(ctx) + if err != nil { + return nil, err + } + + out := map[stores.ID]string{} + for _, st := range l { + out[st.ID] = st.LocalPath + } + + return out, nil +} + +func (m *Manager) FsStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) { + return m.storage.FsStat(ctx, id) +} + +func (m *Manager) SchedDiag(ctx context.Context) (interface{}, error) { + return m.sched.Info(ctx) +} + +func (m *Manager) Close(ctx context.Context) error { + return m.sched.Close(ctx) +} + +var _ SectorManager = &Manager{} diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go new file mode 100644 index 000000000..10e6a5020 --- /dev/null +++ b/extern/sector-storage/manager_test.go @@ -0,0 +1,152 @@ +package sectorstorage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/filecoin-project/sector-storage/sealtasks" + logging "github.com/ipfs/go-log" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/stores" +) + +func init() { + logging.SetAllLoggers(logging.LevelDebug) +} + +type testStorage stores.StorageConfig + +func (t testStorage) DiskUsage(path string) (int64, error) { + return 1, nil // close enough +} + +func newTestStorage(t *testing.T) *testStorage { + tp, err := ioutil.TempDir(os.TempDir(), "sector-storage-test-") + require.NoError(t, err) + + { + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ + ID: stores.ID(uuid.New().String()), + Weight: 1, + CanSeal: true, + CanStore: true, + }, "", " ") + require.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(tp, "sectorstore.json"), b, 0644) + require.NoError(t, err) + } + + return &testStorage{ + StoragePaths: []stores.LocalPath{ + {Path: tp}, + }, + } +} + +func (t testStorage) cleanup() { + for _, path := range t.StoragePaths { + if err := os.RemoveAll(path.Path); err != nil { + fmt.Println("Cleanup error:", err) + } + } +} + +func (t testStorage) GetStorage() (stores.StorageConfig, error) { + return stores.StorageConfig(t), nil +} + +func (t *testStorage) SetStorage(f func(*stores.StorageConfig)) error { + f((*stores.StorageConfig)(t)) + return nil +} + +func (t *testStorage) Stat(path string) (fsutil.FsStat, error) { + return fsutil.Statfs(path) +} + +var _ stores.LocalStorage = &testStorage{} + +func newTestMgr(ctx context.Context, t *testing.T) (*Manager, *stores.Local, *stores.Remote, *stores.Index) { + st := newTestStorage(t) + defer st.cleanup() + + si := stores.NewIndex() + cfg := &ffiwrapper.Config{ + SealProofType: abi.RegisteredSealProof_StackedDrg2KiBV1, + } + + lstor, err := stores.NewLocal(ctx, st, si, nil) + require.NoError(t, err) + + prover, err := ffiwrapper.New(&readonlyProvider{stor: lstor}, cfg) + require.NoError(t, err) + + stor := stores.NewRemote(lstor, si, nil, 6000) + + m := &Manager{ + scfg: cfg, + + ls: st, + storage: stor, + localStore: lstor, + remoteHnd: &stores.FetchHandler{Local: lstor}, + index: si, + + sched: newScheduler(cfg.SealProofType), + + Prover: prover, + } + + go m.sched.runSched() + + return m, lstor, stor, si +} + +func TestSimple(t *testing.T) { + logging.SetAllLoggers(logging.LevelDebug) + + ctx := context.Background() + m, lstor, _, _ := newTestMgr(ctx, t) + + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTFetch, + } + + err := m.AddWorker(ctx, newTestWorker(WorkerConfig{ + SealProof: abi.RegisteredSealProof_StackedDrg2KiBV1, + TaskTypes: localTasks, + }, lstor)) + require.NoError(t, err) + + sid := abi.SectorID{Miner: 1000, Number: 1} + + pi, err := m.AddPiece(ctx, sid, nil, 1016, strings.NewReader(strings.Repeat("testthis", 127))) + require.NoError(t, err) + require.Equal(t, abi.PaddedPieceSize(1024), pi.Size) + + piz, err := m.AddPiece(ctx, sid, nil, 1016, bytes.NewReader(make([]byte, 1016)[:])) + require.NoError(t, err) + require.Equal(t, abi.PaddedPieceSize(1024), piz.Size) + + pieces := []abi.PieceInfo{pi, piz} + + ticket := abi.SealRandomness{9, 9, 9, 9, 9, 9, 9, 9} + + _, err = m.SealPreCommit1(ctx, sid, ticket, pieces) + require.NoError(t, err) + +} diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go new file mode 100644 index 000000000..6eb71cd6b --- /dev/null +++ b/extern/sector-storage/mock/mock.go @@ -0,0 +1,458 @@ +package mock + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "sync" + + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" + "golang.org/x/xerrors" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/storiface" +) + +var log = logging.Logger("sbmock") + +type SectorMgr struct { + sectors map[abi.SectorID]*sectorState + pieces map[cid.Cid][]byte + sectorSize abi.SectorSize + nextSectorID abi.SectorNumber + proofType abi.RegisteredSealProof + + lk sync.Mutex +} + +type mockVerif struct{} + +func NewMockSectorMgr(ssize abi.SectorSize, genesisSectors []abi.SectorID) *SectorMgr { + rt, err := ffiwrapper.SealProofTypeFromSectorSize(ssize) + if err != nil { + panic(err) + } + + sectors := make(map[abi.SectorID]*sectorState) + for _, sid := range genesisSectors { + sectors[sid] = §orState{ + failed: false, + state: stateCommit, + } + } + + return &SectorMgr{ + sectors: sectors, + pieces: map[cid.Cid][]byte{}, + sectorSize: ssize, + nextSectorID: 5, + proofType: rt, + } +} + +const ( + statePacking = iota + statePreCommit + stateCommit // nolint +) + +type sectorState struct { + pieces []cid.Cid + failed bool + + state int + + lk sync.Mutex +} + +func (mgr *SectorMgr) NewSector(ctx context.Context, sector abi.SectorID) error { + return nil +} + +func (mgr *SectorMgr) AddPiece(ctx context.Context, sectorId abi.SectorID, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { + log.Warn("Add piece: ", sectorId, size, mgr.proofType) + + var b bytes.Buffer + tr := io.TeeReader(r, &b) + + c, err := ffiwrapper.GeneratePieceCIDFromFile(mgr.proofType, tr, size) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("failed to generate piece cid: %w", err) + } + + log.Warn("Generated Piece CID: ", c) + + mgr.lk.Lock() + mgr.pieces[c] = b.Bytes() + + ss, ok := mgr.sectors[sectorId] + if !ok { + ss = §orState{ + state: statePacking, + } + mgr.sectors[sectorId] = ss + } + mgr.lk.Unlock() + + ss.lk.Lock() + ss.pieces = append(ss.pieces, c) + ss.lk.Unlock() + + return abi.PieceInfo{ + + Size: size.Padded(), + PieceCID: c, + }, nil +} + +func (mgr *SectorMgr) SectorSize() abi.SectorSize { + return mgr.sectorSize +} + +func (mgr *SectorMgr) AcquireSectorNumber() (abi.SectorNumber, error) { + mgr.lk.Lock() + defer mgr.lk.Unlock() + id := mgr.nextSectorID + mgr.nextSectorID++ + return id, nil +} + +func (mgr *SectorMgr) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { + mgr.lk.Lock() + ss, ok := mgr.sectors[sid] + mgr.lk.Unlock() + if !ok { + return nil, xerrors.Errorf("no sector with id %d in storage", sid) + } + + ss.lk.Lock() + defer ss.lk.Unlock() + + ussize := abi.PaddedPieceSize(mgr.sectorSize).Unpadded() + + // TODO: verify pieces in sinfo.pieces match passed in pieces + + var sum abi.UnpaddedPieceSize + for _, p := range pieces { + sum += p.Size.Unpadded() + } + + if sum != ussize { + return nil, xerrors.Errorf("aggregated piece sizes don't match up: %d != %d", sum, ussize) + } + + if ss.state != statePacking { + return nil, xerrors.Errorf("cannot call pre-seal on sector not in 'packing' state") + } + + opFinishWait(ctx) + + ss.state = statePreCommit + + pis := make([]abi.PieceInfo, len(ss.pieces)) + for i, piece := range ss.pieces { + pis[i] = abi.PieceInfo{ + Size: pieces[i].Size, + PieceCID: piece, + } + } + + commd, err := MockVerifier.GenerateDataCommitment(mgr.proofType, pis) + if err != nil { + return nil, err + } + + _, _, cc, err := commcid.CIDToCommitment(commd) + if err != nil { + panic(err) + } + + cc[0] ^= 'd' + + return cc, nil +} + +func (mgr *SectorMgr) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { + db := []byte(string(phase1Out)) + db[0] ^= 'd' + + d, _ := commcid.DataCommitmentV1ToCID(db) + + commr := make([]byte, 32) + for i := range db { + commr[32-(i+1)] = db[i] + } + + commR, _ := commcid.ReplicaCommitmentV1ToCID(commr) + + return storage.SectorCids{ + Unsealed: d, + Sealed: commR, + }, nil +} + +func (mgr *SectorMgr) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { + mgr.lk.Lock() + ss, ok := mgr.sectors[sid] + mgr.lk.Unlock() + if !ok { + return nil, xerrors.Errorf("no such sector %d", sid) + } + ss.lk.Lock() + defer ss.lk.Unlock() + + if ss.failed { + return nil, xerrors.Errorf("[mock] cannot commit failed sector %d", sid) + } + + if ss.state != statePreCommit { + return nil, xerrors.Errorf("cannot commit sector that has not been precommitted") + } + + opFinishWait(ctx) + + var out [32]byte + for i := range out { + out[i] = cids.Unsealed.Bytes()[i] + cids.Sealed.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid.Number&0xff) + } + + return out[:], nil +} + +func (mgr *SectorMgr) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { + var out [32]byte + for i := range out { + out[i] = phase1Out[i] ^ byte(sid.Number&0xff) + } + + return out[:], nil +} + +// Test Instrumentation Methods + +func (mgr *SectorMgr) MarkFailed(sid abi.SectorID, failed bool) error { + mgr.lk.Lock() + defer mgr.lk.Unlock() + ss, ok := mgr.sectors[sid] + if !ok { + return fmt.Errorf("no such sector in storage") + } + + ss.failed = failed + return nil +} + +func opFinishWait(ctx context.Context) { + val, ok := ctx.Value("opfinish").(chan struct{}) + if !ok { + return + } + <-val +} + +func AddOpFinish(ctx context.Context) (context.Context, func()) { + done := make(chan struct{}) + + return context.WithValue(ctx, "opfinish", done), func() { + close(done) + } +} + +func (mgr *SectorMgr) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, error) { + return generateFakePoSt(sectorInfo, abi.RegisteredSealProof.RegisteredWinningPoStProof, randomness), nil +} + +func (mgr *SectorMgr) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, []abi.SectorID, error) { + si := make([]abi.SectorInfo, 0, len(sectorInfo)) + var skipped []abi.SectorID + + for _, info := range sectorInfo { + sid := abi.SectorID{ + Miner: minerID, + Number: info.SectorNumber, + } + + _, found := mgr.sectors[sid] + + if found && !mgr.sectors[sid].failed { + si = append(si, info) + } else { + skipped = append(skipped, sid) + } + } + + return generateFakePoSt(si, abi.RegisteredSealProof.RegisteredWindowPoStProof, randomness), skipped, nil +} + +func generateFakePoSt(sectorInfo []abi.SectorInfo, rpt func(abi.RegisteredSealProof) (abi.RegisteredPoStProof, error), randomness abi.PoStRandomness) []abi.PoStProof { + sectors := abi.NewBitField() + for _, info := range sectorInfo { + sectors.Set(uint64(info.SectorNumber)) + } + + wp, err := rpt(sectorInfo[0].SealProof) + if err != nil { + panic(err) + } + + var proofBuf bytes.Buffer + + _, err = proofBuf.Write(randomness) + if err != nil { + panic(err) + } + + if err := sectors.MarshalCBOR(&proofBuf); err != nil { + panic(err) + } + + return []abi.PoStProof{ + { + PoStProof: wp, + ProofBytes: proofBuf.Bytes(), + }, + } +} + +func (mgr *SectorMgr) ReadPiece(ctx context.Context, w io.Writer, sectorID abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, c cid.Cid) error { + if len(mgr.sectors[sectorID].pieces) > 1 || offset != 0 { + panic("implme") + } + + _, err := io.CopyN(w, bytes.NewReader(mgr.pieces[mgr.sectors[sectorID].pieces[0]]), int64(size)) + return err +} + +func (mgr *SectorMgr) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) { + usize := abi.PaddedPieceSize(mgr.sectorSize).Unpadded() + sid, err := mgr.AcquireSectorNumber() + if err != nil { + return abi.SectorID{}, nil, err + } + + buf := make([]byte, usize) + rand.Read(buf) + + id := abi.SectorID{ + Miner: mid, + Number: sid, + } + + pi, err := mgr.AddPiece(context.TODO(), id, nil, usize, bytes.NewReader(buf)) + if err != nil { + return abi.SectorID{}, nil, err + } + + return id, []abi.PieceInfo{pi}, nil +} + +func (mgr *SectorMgr) FinalizeSector(context.Context, abi.SectorID, []storage.Range) error { + return nil +} + +func (mgr *SectorMgr) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { + return nil +} + +func (mgr *SectorMgr) Remove(ctx context.Context, sector abi.SectorID) error { + mgr.lk.Lock() + defer mgr.lk.Unlock() + + if _, has := mgr.sectors[sector]; !has { + return xerrors.Errorf("sector not found") + } + + delete(mgr.sectors, sector) + return nil +} + +func (mgr *SectorMgr) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof, ids []abi.SectorID) ([]abi.SectorID, error) { + var bad []abi.SectorID + + for _, sid := range ids { + _, found := mgr.sectors[sid] + + if !found || mgr.sectors[sid].failed { + bad = append(bad, sid) + } + } + + return bad, nil +} + +func (m mockVerif) VerifySeal(svi abi.SealVerifyInfo) (bool, error) { + if len(svi.Proof) != 32 { // Real ones are longer, but this should be fine + return false, nil + } + + for i, b := range svi.Proof { + if b != svi.UnsealedCID.Bytes()[i]+svi.SealedCID.Bytes()[31-i]-svi.InteractiveRandomness[i]*svi.Randomness[i] { + return false, nil + } + } + + return true, nil +} + +func (m mockVerif) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) { + return true, nil +} + +func (m mockVerif) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) { + if len(info.Proofs) != 1 { + return false, xerrors.Errorf("expected 1 proof entry") + } + + proof := info.Proofs[0] + + if !bytes.Equal(proof.ProofBytes[:len(info.Randomness)], info.Randomness) { + return false, xerrors.Errorf("bad randomness") + } + + sectors := abi.NewBitField() + if err := sectors.UnmarshalCBOR(bytes.NewReader(proof.ProofBytes[len(info.Randomness):])); err != nil { + return false, xerrors.Errorf("unmarshaling sectors bitfield from \"proof\": %w", err) + } + + challenged := abi.NewBitField() + for _, sector := range info.ChallengedSectors { + challenged.Set(uint64(sector.SectorNumber)) + } + + { + b1, err := sectors.MarshalJSON() + if err != nil { + return false, err + } + + b2, err := challenged.MarshalJSON() + if err != nil { + return false, err + } + + if !bytes.Equal(b1, b2) { + return false, xerrors.Errorf("proven and challenged sector sets didn't match: %s != !s", string(b1), string(b2)) + } + } + + return true, nil +} + +func (m mockVerif) GenerateDataCommitment(pt abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { + return ffiwrapper.GenerateUnsealedCID(pt, pieces) +} + +func (m mockVerif) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, eligibleSectorCount uint64) ([]uint64, error) { + return []uint64{0}, nil +} + +var MockVerifier = mockVerif{} + +var _ storage.Sealer = &SectorMgr{} +var _ ffiwrapper.Verifier = MockVerifier diff --git a/extern/sector-storage/mock/mock_test.go b/extern/sector-storage/mock/mock_test.go new file mode 100644 index 000000000..c7d43e8b9 --- /dev/null +++ b/extern/sector-storage/mock/mock_test.go @@ -0,0 +1,45 @@ +package mock + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +func TestOpFinish(t *testing.T) { + sb := NewMockSectorMgr(2048, nil) + + sid, pieces, err := sb.StageFakeData(123) + if err != nil { + t.Fatal(err) + } + + ctx, done := AddOpFinish(context.TODO()) + + finished := make(chan struct{}) + go func() { + _, err := sb.SealPreCommit1(ctx, sid, abi.SealRandomness{}, pieces) + if err != nil { + t.Error(err) + return + } + + close(finished) + }() + + select { + case <-finished: + t.Fatal("should not finish until we tell it to") + case <-time.After(time.Second / 2): + } + + done() + + select { + case <-finished: + case <-time.After(time.Second / 2): + t.Fatal("should finish after we tell it to") + } +} diff --git a/extern/sector-storage/mock/util.go b/extern/sector-storage/mock/util.go new file mode 100644 index 000000000..2d2ebbfe2 --- /dev/null +++ b/extern/sector-storage/mock/util.go @@ -0,0 +1,9 @@ +package mock + +func CommDR(in []byte) (out [32]byte) { + for i, b := range in { + out[i] = ^b + } + + return out +} diff --git a/extern/sector-storage/parameters.json b/extern/sector-storage/parameters.json new file mode 100644 index 000000000..b632c17e8 --- /dev/null +++ b/extern/sector-storage/parameters.json @@ -0,0 +1,152 @@ +{ + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": { + "cid": "QmeDRyxek34F1H6xJY6AkFdWvPsy5F6dKTrebV3ZtWT4ky", + "digest": "f5827f2d8801c62c831e0f972f6dc8bb", + "sector_size": 2048 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": { + "cid": "QmUw1ZmG4BBbX19MsbH3zAEGKUc42iFJc5ZAyomDHeJTsA", + "digest": "398fecdb4b2de445125852bc3c080b35", + "sector_size": 2048 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": { + "cid": "QmUeNKp9YZpiAFm81RV5KuxH1FDGJx2DuwcbU2XNSZLLSv", + "digest": "2b6d2972ac9e862e8134d98fb695b0c5", + "sector_size": 536870912 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": { + "cid": "QmQaQmTXX995Akd66ggtJY5bNx6Gkxk8P34JTdMMq8393G", + "digest": "3688c9eb256b7b17f411dad78d5ef74a", + "sector_size": 536870912 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": { + "cid": "QmfEYTMSkwGJTumQx26iKXGNKiYh3mmAC4SkdybZpJCj5p", + "digest": "09bff16aed893349d94485cfae366a9c", + "sector_size": 2048 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": { + "cid": "QmP4ThPieSUJyRanjibWpT5R5cCMzMAU4j8Y7kBn7CSW1Q", + "digest": "142f2f7e8f1b1779290315cabfd2c803", + "sector_size": 2048 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": { + "cid": "QmcAixrHsz29DgvtZiMc2kQjvPRvWxYUp36QYmRDZbmREm", + "digest": "8f987f64d434365562180b96ec12e299", + "sector_size": 8388608 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": { + "cid": "QmT4iFnbL6r4txS5PXsiV7NTzbhCxHy54PvdkJJGV2VFXb", + "digest": "94b6c24ac01924f4feeecedd16b5d77d", + "sector_size": 8388608 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": { + "cid": "QmbjFst6SFCK1KsTQrfwPdxf3VTNa1raed574tEZZ9PoyQ", + "digest": "2c245fe8179839dd6c6cdea207c67ae8", + "sector_size": 8388608 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": { + "cid": "QmQJKmvZN1a5cQ1Nw6CDyXs3nuRPzvyU5NvCFMUL2BfcZC", + "digest": "56ae47bfda53bb8d22981ed8d8d27d72", + "sector_size": 8388608 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": { + "cid": "QmQCABxeTpdvXTyjDyk7nPBxkQzCh7MXfGztWnSXEPKMLW", + "digest": "7e6b2eb5ecbb11ac651ad66ebbb2075a", + "sector_size": 536870912 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": { + "cid": "QmPBweyugh5Sx4umk8ULhgEGbjY8xmWLfU6M7EMpc8Mad6", + "digest": "94a8d9e25a9ab9674d339833664eba25", + "sector_size": 536870912 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": { + "cid": "QmY5yax1E9KymBnCeHksE9Zi8NieZbmwcpoDGoabkeeb9h", + "digest": "c909ea9e3fe25ab9b391a64593afdbba", + "sector_size": 34359738368 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": { + "cid": "QmXnPo4yH5mwMguwrvqgRfduSttbmPrXtbBfbwU21wQWHt", + "digest": "caf900461e988bbf86dbcaca087b7864", + "sector_size": 34359738368 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": { + "cid": "QmZtzzPWwmZEgR7MSMvXRbt9KVK8k4XZ5RLWHybHJW9SdE", + "digest": "a2844f0703f186d143a06146a04577d8", + "sector_size": 34359738368 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": { + "cid": "QmWxEA7EdQCUJTzjNpxg5XTF45D2uVyYnN1QRUb5TRYU8M", + "digest": "2306247a1e616dbe07f01b88196c2044", + "sector_size": 34359738368 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": { + "cid": "QmP676KwuvyF9Y64uJnXvLtvD1xcuWQ6wD23RzYtQ6dd4f", + "digest": "215b1c667a4f46a1d0178338df568615", + "sector_size": 68719476736 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": { + "cid": "QmPvPwbJtcSGyqB1rQJhSF5yvFbX9ZBSsHVej5F8JUyHUJ", + "digest": "0c9c423b28b1455fcbc329a1045fd4dd", + "sector_size": 68719476736 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": { + "cid": "QmUxPQfvckzm1t6MFRdDZ1fDK5UJzAjK7pTZ97cwyachdr", + "digest": "965132f51ae445b0e6d32692b7561995", + "sector_size": 68719476736 + }, + "v27-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": { + "cid": "QmTxq2EBnQWb5R8tS4MHdchj4vNfLYGoSXxwJFvs5xgW4K", + "digest": "fc8c3d26e0e56373ad96cb41520d55a6", + "sector_size": 68719476736 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": { + "cid": "QmRjgZHERgqGoRagR788Kh6ybi26csVYa8mqbqhmZm57Jx", + "digest": "cfc7b0897d1eee48c586f7beb89e67f7", + "sector_size": 2048 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": { + "cid": "QmNjvnvFP7KgovHUddULoB19fBHT81iz7NcUbzEHZUUPsm", + "digest": "fb59bd061c987eac7068008c44de346b", + "sector_size": 2048 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": { + "cid": "QmTpRPBA4dt8fgGpcVzi4L1KA1U2eBHCE8WVmS2GUygMvT", + "digest": "36d465915b0afbf96bd08e7915e00952", + "sector_size": 536870912 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": { + "cid": "QmRzDyVfQCLsxspoVsed5bcQRsG6KiktngJfcNBL3TJPZe", + "digest": "99d16df0eb6a7e227a4f4570c4f6b6f1", + "sector_size": 536870912 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": { + "cid": "QmV8ZjTSGzDUWmFvsq9NSyPBR7eDDUcvCPNgj2yE7HMAFu", + "digest": "34f3ddf1d1c9f41c0cd73b91e8b4bc27", + "sector_size": 8388608 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": { + "cid": "QmTa3VbjTiqJWU6r4WKayaQrUaaBsrpp5UDqYvPDd2C5hs", + "digest": "ec62d59651daa5631d3d1e9c782dd940", + "sector_size": 8388608 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": { + "cid": "Qmf8ngfArxrv9tFWDqBcNegdBMymvuakwyHKd1pbW3pbsb", + "digest": "a16d6f4c6424fb280236739f84b24f97", + "sector_size": 34359738368 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": { + "cid": "QmfQgVFerArJ6Jupwyc9tKjLD9n1J9ajLHBdpY465tRM7M", + "digest": "7a139d82b8a02e35279d657e197f5c1f", + "sector_size": 34359738368 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": { + "cid": "QmfDha8271nXJn14Aq3qQeghjMBWbs6HNSGa6VuzCVk4TW", + "digest": "5d3cd3f107a3bea8a96d1189efd2965c", + "sector_size": 68719476736 + }, + "v27-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": { + "cid": "QmRVtTtiFzHJTHurYzaCvetGAchux9cktixT4aGHthN6Zt", + "digest": "62c366405404e60f171e661492740b1c", + "sector_size": 68719476736 + } +} \ No newline at end of file diff --git a/extern/sector-storage/request_queue.go b/extern/sector-storage/request_queue.go new file mode 100644 index 000000000..85d3abf46 --- /dev/null +++ b/extern/sector-storage/request_queue.go @@ -0,0 +1,45 @@ +package sectorstorage + +import "sort" + +type requestQueue []*workerRequest + +func (q requestQueue) Len() int { return len(q) } + +func (q requestQueue) Less(i, j int) bool { + if q[i].priority != q[j].priority { + return q[i].priority > q[j].priority + } + + if q[i].taskType != q[j].taskType { + return q[i].taskType.Less(q[j].taskType) + } + + return q[i].sector.Number < q[j].sector.Number // optimize minerActor.NewSectors bitfield +} + +func (q requestQueue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] + q[i].index = i + q[j].index = j +} + +func (q *requestQueue) Push(x *workerRequest) { + n := len(*q) + item := x + item.index = n + *q = append(*q, item) + sort.Sort(q) +} + +func (q *requestQueue) Remove(i int) *workerRequest { + old := *q + n := len(old) + item := old[i] + old[i] = old[n-1] + old[n-1] = nil + item.index = -1 + *q = old[0 : n-1] + sort.Sort(q) + return item +} diff --git a/extern/sector-storage/request_queue_test.go b/extern/sector-storage/request_queue_test.go new file mode 100644 index 000000000..cb4a5d5dd --- /dev/null +++ b/extern/sector-storage/request_queue_test.go @@ -0,0 +1,62 @@ +package sectorstorage + +import ( + "fmt" + "testing" + + "github.com/filecoin-project/sector-storage/sealtasks" +) + +func TestRequestQueue(t *testing.T) { + rq := &requestQueue{} + + rq.Push(&workerRequest{taskType: sealtasks.TTAddPiece}) + rq.Push(&workerRequest{taskType: sealtasks.TTPreCommit1}) + rq.Push(&workerRequest{taskType: sealtasks.TTPreCommit2}) + rq.Push(&workerRequest{taskType: sealtasks.TTPreCommit1}) + rq.Push(&workerRequest{taskType: sealtasks.TTAddPiece}) + + dump := func(s string) { + fmt.Println("---") + fmt.Println(s) + + for sqi := 0; sqi < rq.Len(); sqi++ { + task := (*rq)[sqi] + fmt.Println(sqi, task.taskType) + } + } + + dump("start") + + pt := rq.Remove(0) + + dump("pop 1") + + if pt.taskType != sealtasks.TTPreCommit2 { + t.Error("expected precommit2, got", pt.taskType) + } + + pt = rq.Remove(0) + + dump("pop 2") + + if pt.taskType != sealtasks.TTPreCommit1 { + t.Error("expected precommit1, got", pt.taskType) + } + + pt = rq.Remove(1) + + dump("pop 3") + + if pt.taskType != sealtasks.TTAddPiece { + t.Error("expected addpiece, got", pt.taskType) + } + + pt = rq.Remove(0) + + dump("pop 4") + + if pt.taskType != sealtasks.TTPreCommit1 { + t.Error("expected precommit1, got", pt.taskType) + } +} diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go new file mode 100644 index 000000000..2f67dc84e --- /dev/null +++ b/extern/sector-storage/resources.go @@ -0,0 +1,293 @@ +package sectorstorage + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/sealtasks" +) + +type Resources struct { + MinMemory uint64 // What Must be in RAM for decent perf + MaxMemory uint64 // Memory required (swap + ram) + + Threads int // -1 = multithread + CanGPU bool + + BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) +} + +func (r Resources) MultiThread() bool { + return r.Threads == -1 +} + +var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ + sealtasks.TTAddPiece: { + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ // This is probably a bit conservative + MaxMemory: 64 << 30, + MinMemory: 64 << 30, + + Threads: 1, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ // This is probably a bit conservative + MaxMemory: 32 << 30, + MinMemory: 32 << 30, + + Threads: 1, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + Threads: 1, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 2 << 10, + MinMemory: 2 << 10, + + Threads: 1, + + BaseMinMemory: 2 << 10, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 8 << 20, + MinMemory: 8 << 20, + + Threads: 1, + + BaseMinMemory: 8 << 20, + }, + }, + sealtasks.TTPreCommit1: { + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ + MaxMemory: 128 << 30, + MinMemory: 112 << 30, + + Threads: 1, + + BaseMinMemory: 10 << 20, + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ + MaxMemory: 64 << 30, + MinMemory: 56 << 30, + + Threads: 1, + + BaseMinMemory: 10 << 20, + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 1 << 30, + MinMemory: 768 << 20, + + Threads: 1, + + BaseMinMemory: 1 << 20, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 2 << 10, + MinMemory: 2 << 10, + + Threads: 1, + + BaseMinMemory: 2 << 10, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 8 << 20, + MinMemory: 8 << 20, + + Threads: 1, + + BaseMinMemory: 8 << 20, + }, + }, + sealtasks.TTPreCommit2: { + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ + MaxMemory: 64 << 30, + MinMemory: 64 << 30, + + Threads: -1, + CanGPU: true, + + BaseMinMemory: 60 << 30, + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ + MaxMemory: 32 << 30, + MinMemory: 32 << 30, + + Threads: -1, + CanGPU: true, + + BaseMinMemory: 30 << 30, + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 3 << 29, // 1.5G + MinMemory: 1 << 30, + + Threads: -1, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 2 << 10, + MinMemory: 2 << 10, + + Threads: -1, + + BaseMinMemory: 2 << 10, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 8 << 20, + MinMemory: 8 << 20, + + Threads: -1, + + BaseMinMemory: 8 << 20, + }, + }, + sealtasks.TTCommit1: { // Very short (~100ms), so params are very light + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + Threads: 0, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + Threads: 0, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + Threads: 0, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 2 << 10, + MinMemory: 2 << 10, + + Threads: 0, + + BaseMinMemory: 2 << 10, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 8 << 20, + MinMemory: 8 << 20, + + Threads: 0, + + BaseMinMemory: 8 << 20, + }, + }, + sealtasks.TTCommit2: { + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ + MaxMemory: 190 << 30, // TODO: Confirm + MinMemory: 60 << 30, + + Threads: -1, + CanGPU: true, + + BaseMinMemory: 64 << 30, // params + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ + MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory + MinMemory: 30 << 30, + + Threads: -1, + CanGPU: true, + + BaseMinMemory: 32 << 30, // params + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 3 << 29, // 1.5G + MinMemory: 1 << 30, + + Threads: 1, // This is fine + CanGPU: true, + + BaseMinMemory: 10 << 30, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 2 << 10, + MinMemory: 2 << 10, + + Threads: 1, + CanGPU: true, + + BaseMinMemory: 2 << 10, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 8 << 20, + MinMemory: 8 << 20, + + Threads: 1, + CanGPU: true, + + BaseMinMemory: 8 << 20, + }, + }, + sealtasks.TTFetch: { + abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ + MaxMemory: 1 << 20, + MinMemory: 1 << 20, + + Threads: 0, + CanGPU: false, + + BaseMinMemory: 0, + }, + abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ + MaxMemory: 1 << 20, + MinMemory: 1 << 20, + + Threads: 0, + CanGPU: false, + + BaseMinMemory: 0, + }, + abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ + MaxMemory: 1 << 20, + MinMemory: 1 << 20, + + Threads: 0, + CanGPU: false, + + BaseMinMemory: 0, + }, + abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{ + MaxMemory: 1 << 20, + MinMemory: 1 << 20, + + Threads: 0, + CanGPU: false, + + BaseMinMemory: 0, + }, + abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{ + MaxMemory: 1 << 20, + MinMemory: 1 << 20, + + Threads: 0, + CanGPU: false, + + BaseMinMemory: 0, + }, + }, +} + +func init() { + ResourceTable[sealtasks.TTUnseal] = ResourceTable[sealtasks.TTPreCommit1] // TODO: measure accurately + ResourceTable[sealtasks.TTReadUnsealed] = ResourceTable[sealtasks.TTFetch] +} diff --git a/extern/sector-storage/roprov.go b/extern/sector-storage/roprov.go new file mode 100644 index 000000000..fc10ebbec --- /dev/null +++ b/extern/sector-storage/roprov.go @@ -0,0 +1,40 @@ +package sectorstorage + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/stores" +) + +type readonlyProvider struct { + index stores.SectorIndex + stor *stores.Local + spt abi.RegisteredSealProof +} + +func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing stores.PathType) (stores.SectorPaths, func(), error) { + if allocate != stores.FTNone { + return stores.SectorPaths{}, nil, xerrors.New("read-only storage") + } + + ctx, cancel := context.WithCancel(ctx) + + // use TryLock to avoid blocking + locked, err := l.index.StorageTryLock(ctx, id, existing, stores.FTNone) + if err != nil { + cancel() + return stores.SectorPaths{}, nil, xerrors.Errorf("acquiring sector lock: %w", err) + } + if !locked { + cancel() + return stores.SectorPaths{}, nil, xerrors.Errorf("failed to acquire sector lock") + } + + p, _, err := l.stor.AcquireSector(ctx, id, l.spt, existing, allocate, sealing, stores.AcquireMove) + + return p, cancel, err +} diff --git a/extern/sector-storage/sched.go b/extern/sector-storage/sched.go new file mode 100644 index 000000000..e8eda4834 --- /dev/null +++ b/extern/sector-storage/sched.go @@ -0,0 +1,703 @@ +package sectorstorage + +import ( + "context" + "fmt" + "math/rand" + "sort" + "sync" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/storiface" +) + +type schedPrioCtxKey int + +var SchedPriorityKey schedPrioCtxKey +var DefaultSchedPriority = 0 +var SelectorTimeout = 5 * time.Second + +var ( + SchedWindows = 2 +) + +func getPriority(ctx context.Context) int { + sp := ctx.Value(SchedPriorityKey) + if p, ok := sp.(int); ok { + return p + } + + return DefaultSchedPriority +} + +func WithPriority(ctx context.Context, priority int) context.Context { + return context.WithValue(ctx, SchedPriorityKey, priority) +} + +const mib = 1 << 20 + +type WorkerAction func(ctx context.Context, w Worker) error + +type WorkerSelector interface { + Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, a *workerHandle) (bool, error) // true if worker is acceptable for performing a task + + Cmp(ctx context.Context, task sealtasks.TaskType, a, b *workerHandle) (bool, error) // true if a is preferred over b +} + +type scheduler struct { + spt abi.RegisteredSealProof + + workersLk sync.RWMutex + nextWorker WorkerID + workers map[WorkerID]*workerHandle + + newWorkers chan *workerHandle + + watchClosing chan WorkerID + workerClosing chan WorkerID + + schedule chan *workerRequest + windowRequests chan *schedWindowRequest + + // owned by the sh.runSched goroutine + schedQueue *requestQueue + openWindows []*schedWindowRequest + + info chan func(interface{}) + + closing chan struct{} + closed chan struct{} + testSync chan struct{} // used for testing +} + +type workerHandle struct { + w Worker + + info storiface.WorkerInfo + + preparing *activeResources + active *activeResources + + lk sync.Mutex + + // stats / tracking + wt *workTracker + + // for sync manager goroutine closing + cleanupStarted bool + closedMgr chan struct{} + closingMgr chan struct{} +} + +type schedWindowRequest struct { + worker WorkerID + + done chan *schedWindow +} + +type schedWindow struct { + allocated activeResources + todo []*workerRequest +} + +type activeResources struct { + memUsedMin uint64 + memUsedMax uint64 + gpuUsed bool + cpuUse uint64 + + cond *sync.Cond +} + +type workerRequest struct { + sector abi.SectorID + taskType sealtasks.TaskType + priority int // larger values more important + sel WorkerSelector + + prepare WorkerAction + work WorkerAction + + index int // The index of the item in the heap. + + indexHeap int + ret chan<- workerResponse + ctx context.Context +} + +type workerResponse struct { + err error +} + +func newScheduler(spt abi.RegisteredSealProof) *scheduler { + return &scheduler{ + spt: spt, + + nextWorker: 0, + workers: map[WorkerID]*workerHandle{}, + + newWorkers: make(chan *workerHandle), + + watchClosing: make(chan WorkerID), + workerClosing: make(chan WorkerID), + + schedule: make(chan *workerRequest), + windowRequests: make(chan *schedWindowRequest), + + schedQueue: &requestQueue{}, + + info: make(chan func(interface{})), + + closing: make(chan struct{}), + closed: make(chan struct{}), + } +} + +func (sh *scheduler) Schedule(ctx context.Context, sector abi.SectorID, taskType sealtasks.TaskType, sel WorkerSelector, prepare WorkerAction, work WorkerAction) error { + ret := make(chan workerResponse) + + select { + case sh.schedule <- &workerRequest{ + sector: sector, + taskType: taskType, + priority: getPriority(ctx), + sel: sel, + + prepare: prepare, + work: work, + + ret: ret, + ctx: ctx, + }: + case <-sh.closing: + return xerrors.New("closing") + case <-ctx.Done(): + return ctx.Err() + } + + select { + case resp := <-ret: + return resp.err + case <-sh.closing: + return xerrors.New("closing") + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *workerRequest) respond(err error) { + select { + case r.ret <- workerResponse{err: err}: + case <-r.ctx.Done(): + log.Warnf("request got cancelled before we could respond") + } +} + +type SchedDiagRequestInfo struct { + Sector abi.SectorID + TaskType sealtasks.TaskType + Priority int +} + +type SchedDiagInfo struct { + Requests []SchedDiagRequestInfo + OpenWindows []WorkerID +} + +func (sh *scheduler) runSched() { + defer close(sh.closed) + + go sh.runWorkerWatcher() + + for { + select { + case w := <-sh.newWorkers: + sh.newWorker(w) + + case wid := <-sh.workerClosing: + sh.dropWorker(wid) + + case req := <-sh.schedule: + sh.schedQueue.Push(req) + sh.trySched() + + if sh.testSync != nil { + sh.testSync <- struct{}{} + } + case req := <-sh.windowRequests: + sh.openWindows = append(sh.openWindows, req) + sh.trySched() + + case ireq := <-sh.info: + ireq(sh.diag()) + + case <-sh.closing: + sh.schedClose() + return + } + } +} + +func (sh *scheduler) diag() SchedDiagInfo { + var out SchedDiagInfo + + for sqi := 0; sqi < sh.schedQueue.Len(); sqi++ { + task := (*sh.schedQueue)[sqi] + + out.Requests = append(out.Requests, SchedDiagRequestInfo{ + Sector: task.sector, + TaskType: task.taskType, + Priority: task.priority, + }) + } + + for _, window := range sh.openWindows { + out.OpenWindows = append(out.OpenWindows, window.worker) + } + + return out +} + +func (sh *scheduler) trySched() { + /* + This assigns tasks to workers based on: + - Task priority (achieved by handling sh.schedQueue in order, since it's already sorted by priority) + - Worker resource availability + - Task-specified worker preference (acceptableWindows array below sorted by this preference) + - Window request age + + 1. For each task in the schedQueue find windows which can handle them + 1.1. Create list of windows capable of handling a task + 1.2. Sort windows according to task selector preferences + 2. Going through schedQueue again, assign task to first acceptable window + with resources available + 3. Submit windows with scheduled tasks to workers + + */ + + windows := make([]schedWindow, len(sh.openWindows)) + acceptableWindows := make([][]int, sh.schedQueue.Len()) + + log.Debugf("SCHED %d queued; %d open windows", sh.schedQueue.Len(), len(windows)) + + sh.workersLk.RLock() + defer sh.workersLk.RUnlock() + + // Step 1 + for sqi := 0; sqi < sh.schedQueue.Len(); sqi++ { + task := (*sh.schedQueue)[sqi] + needRes := ResourceTable[task.taskType][sh.spt] + + task.indexHeap = sqi + for wnd, windowRequest := range sh.openWindows { + worker := sh.workers[windowRequest.worker] + + // TODO: allow bigger windows + if !windows[wnd].allocated.canHandleRequest(needRes, windowRequest.worker, worker.info.Resources) { + continue + } + + rpcCtx, cancel := context.WithTimeout(task.ctx, SelectorTimeout) + ok, err := task.sel.Ok(rpcCtx, task.taskType, sh.spt, worker) + cancel() + if err != nil { + log.Errorf("trySched(1) req.sel.Ok error: %+v", err) + continue + } + + if !ok { + continue + } + + acceptableWindows[sqi] = append(acceptableWindows[sqi], wnd) + } + + if len(acceptableWindows[sqi]) == 0 { + continue + } + + // Pick best worker (shuffle in case some workers are equally as good) + rand.Shuffle(len(acceptableWindows[sqi]), func(i, j int) { + acceptableWindows[sqi][i], acceptableWindows[sqi][j] = acceptableWindows[sqi][j], acceptableWindows[sqi][i] + }) + sort.SliceStable(acceptableWindows[sqi], func(i, j int) bool { + wii := sh.openWindows[acceptableWindows[sqi][i]].worker + wji := sh.openWindows[acceptableWindows[sqi][j]].worker + + if wii == wji { + // for the same worker prefer older windows + return acceptableWindows[sqi][i] < acceptableWindows[sqi][j] + } + + wi := sh.workers[wii] + wj := sh.workers[wji] + + rpcCtx, cancel := context.WithTimeout(task.ctx, SelectorTimeout) + defer cancel() + + r, err := task.sel.Cmp(rpcCtx, task.taskType, wi, wj) + if err != nil { + log.Error("selecting best worker: %s", err) + } + return r + }) + } + + log.Debugf("SCHED windows: %+v", windows) + log.Debugf("SCHED Acceptable win: %+v", acceptableWindows) + + // Step 2 + scheduled := 0 + + for sqi := 0; sqi < sh.schedQueue.Len(); sqi++ { + task := (*sh.schedQueue)[sqi] + needRes := ResourceTable[task.taskType][sh.spt] + + selectedWindow := -1 + for _, wnd := range acceptableWindows[task.indexHeap] { + wid := sh.openWindows[wnd].worker + wr := sh.workers[wid].info.Resources + + log.Debugf("SCHED try assign sqi:%d sector %d to window %d", sqi, task.sector.Number, wnd) + + // TODO: allow bigger windows + if !windows[wnd].allocated.canHandleRequest(needRes, wid, wr) { + continue + } + + log.Debugf("SCHED ASSIGNED sqi:%d sector %d to window %d", sqi, task.sector.Number, wnd) + + windows[wnd].allocated.add(wr, needRes) + + selectedWindow = wnd + break + } + + if selectedWindow < 0 { + // all windows full + continue + } + + windows[selectedWindow].todo = append(windows[selectedWindow].todo, task) + + sh.schedQueue.Remove(sqi) + sqi-- + scheduled++ + } + + // Step 3 + + if scheduled == 0 { + return + } + + scheduledWindows := map[int]struct{}{} + for wnd, window := range windows { + if len(window.todo) == 0 { + // Nothing scheduled here, keep the window open + continue + } + + scheduledWindows[wnd] = struct{}{} + + window := window // copy + select { + case sh.openWindows[wnd].done <- &window: + default: + log.Error("expected sh.openWindows[wnd].done to be buffered") + } + } + + // Rewrite sh.openWindows array, removing scheduled windows + newOpenWindows := make([]*schedWindowRequest, 0, len(sh.openWindows)-len(scheduledWindows)) + for wnd, window := range sh.openWindows { + if _, scheduled := scheduledWindows[wnd]; scheduled { + // keep unscheduled windows open + continue + } + + newOpenWindows = append(newOpenWindows, window) + } + + sh.openWindows = newOpenWindows +} + +func (sh *scheduler) runWorker(wid WorkerID) { + var ready sync.WaitGroup + ready.Add(1) + defer ready.Wait() + + go func() { + sh.workersLk.RLock() + worker, found := sh.workers[wid] + sh.workersLk.RUnlock() + + ready.Done() + + if !found { + panic(fmt.Sprintf("worker %d not found", wid)) + } + + defer close(worker.closedMgr) + + scheduledWindows := make(chan *schedWindow, SchedWindows) + taskDone := make(chan struct{}, 1) + windowsRequested := 0 + + var activeWindows []*schedWindow + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + workerClosing, err := worker.w.Closing(ctx) + if err != nil { + return + } + + defer func() { + log.Warnw("Worker closing", "workerid", wid) + + // TODO: close / return all queued tasks + }() + + for { + // ask for more windows if we need them + for ; windowsRequested < SchedWindows; windowsRequested++ { + select { + case sh.windowRequests <- &schedWindowRequest{ + worker: wid, + done: scheduledWindows, + }: + case <-sh.closing: + return + case <-workerClosing: + return + case <-worker.closingMgr: + return + } + } + + select { + case w := <-scheduledWindows: + activeWindows = append(activeWindows, w) + case <-taskDone: + log.Debugw("task done", "workerid", wid) + case <-sh.closing: + return + case <-workerClosing: + return + case <-worker.closingMgr: + return + } + + assignLoop: + // process windows in order + for len(activeWindows) > 0 { + // process tasks within a window in order + for len(activeWindows[0].todo) > 0 { + todo := activeWindows[0].todo[0] + needRes := ResourceTable[todo.taskType][sh.spt] + + sh.workersLk.RLock() + worker.lk.Lock() + ok := worker.preparing.canHandleRequest(needRes, wid, worker.info.Resources) + worker.lk.Unlock() + + if !ok { + sh.workersLk.RUnlock() + break assignLoop + } + + log.Debugf("assign worker sector %d", todo.sector.Number) + err := sh.assignWorker(taskDone, wid, worker, todo) + sh.workersLk.RUnlock() + + if err != nil { + log.Error("assignWorker error: %+v", err) + go todo.respond(xerrors.Errorf("assignWorker error: %w", err)) + } + + activeWindows[0].todo = activeWindows[0].todo[1:] + } + + copy(activeWindows, activeWindows[1:]) + activeWindows[len(activeWindows)-1] = nil + activeWindows = activeWindows[:len(activeWindows)-1] + + windowsRequested-- + } + } + }() +} + +func (sh *scheduler) assignWorker(taskDone chan struct{}, wid WorkerID, w *workerHandle, req *workerRequest) error { + needRes := ResourceTable[req.taskType][sh.spt] + + w.lk.Lock() + w.preparing.add(w.info.Resources, needRes) + w.lk.Unlock() + + go func() { + err := req.prepare(req.ctx, w.wt.worker(w.w)) + sh.workersLk.Lock() + + if err != nil { + w.lk.Lock() + w.preparing.free(w.info.Resources, needRes) + w.lk.Unlock() + sh.workersLk.Unlock() + + select { + case taskDone <- struct{}{}: + case <-sh.closing: + log.Warnf("scheduler closed while sending response (prepare error: %+v)", err) + } + + select { + case req.ret <- workerResponse{err: err}: + case <-req.ctx.Done(): + log.Warnf("request got cancelled before we could respond (prepare error: %+v)", err) + case <-sh.closing: + log.Warnf("scheduler closed while sending response (prepare error: %+v)", err) + } + return + } + + err = w.active.withResources(wid, w.info.Resources, needRes, &sh.workersLk, func() error { + w.lk.Lock() + w.preparing.free(w.info.Resources, needRes) + w.lk.Unlock() + sh.workersLk.Unlock() + defer sh.workersLk.Lock() // we MUST return locked from this function + + select { + case taskDone <- struct{}{}: + case <-sh.closing: + } + + err = req.work(req.ctx, w.wt.worker(w.w)) + + select { + case req.ret <- workerResponse{err: err}: + case <-req.ctx.Done(): + log.Warnf("request got cancelled before we could respond") + case <-sh.closing: + log.Warnf("scheduler closed while sending response") + } + + return nil + }) + + sh.workersLk.Unlock() + + // This error should always be nil, since nothing is setting it, but just to be safe: + if err != nil { + log.Errorf("error executing worker (withResources): %+v", err) + } + }() + + return nil +} + +func (sh *scheduler) newWorker(w *workerHandle) { + w.closedMgr = make(chan struct{}) + w.closingMgr = make(chan struct{}) + + sh.workersLk.Lock() + + id := sh.nextWorker + sh.workers[id] = w + sh.nextWorker++ + + sh.workersLk.Unlock() + + sh.runWorker(id) + + select { + case sh.watchClosing <- id: + case <-sh.closing: + return + } +} + +func (sh *scheduler) dropWorker(wid WorkerID) { + sh.workersLk.Lock() + defer sh.workersLk.Unlock() + + w := sh.workers[wid] + + sh.workerCleanup(wid, w) + + delete(sh.workers, wid) +} + +func (sh *scheduler) workerCleanup(wid WorkerID, w *workerHandle) { + if !w.cleanupStarted { + close(w.closingMgr) + } + select { + case <-w.closedMgr: + case <-time.After(time.Second): + log.Errorf("timeout closing worker manager goroutine %d", wid) + } + + if !w.cleanupStarted { + w.cleanupStarted = true + + newWindows := make([]*schedWindowRequest, 0, len(sh.openWindows)) + for _, window := range sh.openWindows { + if window.worker != wid { + newWindows = append(newWindows, window) + } + } + sh.openWindows = newWindows + + log.Debugf("dropWorker %d", wid) + + go func() { + if err := w.w.Close(); err != nil { + log.Warnf("closing worker %d: %+v", err) + } + }() + } +} + +func (sh *scheduler) schedClose() { + sh.workersLk.Lock() + defer sh.workersLk.Unlock() + log.Debugf("closing scheduler") + + for i, w := range sh.workers { + sh.workerCleanup(i, w) + } +} + +func (sh *scheduler) Info(ctx context.Context) (interface{}, error) { + ch := make(chan interface{}, 1) + + sh.info <- func(res interface{}) { + ch <- res + } + + select { + case res := <-ch: + return res, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (sh *scheduler) Close(ctx context.Context) error { + close(sh.closing) + select { + case <-sh.closed: + case <-ctx.Done(): + return ctx.Err() + } + return nil +} diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go new file mode 100644 index 000000000..0ba9d1f66 --- /dev/null +++ b/extern/sector-storage/sched_resources.go @@ -0,0 +1,110 @@ +package sectorstorage + +import ( + "sync" + + "github.com/filecoin-project/sector-storage/storiface" +) + +func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error { + for !a.canHandleRequest(r, id, wr) { + if a.cond == nil { + a.cond = sync.NewCond(locker) + } + a.cond.Wait() + } + + a.add(wr, r) + + err := cb() + + a.free(wr, r) + if a.cond != nil { + a.cond.Broadcast() + } + + return err +} + +func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { + a.gpuUsed = r.CanGPU + if r.MultiThread() { + a.cpuUse += wr.CPUs + } else { + a.cpuUse += uint64(r.Threads) + } + + a.memUsedMin += r.MinMemory + a.memUsedMax += r.MaxMemory +} + +func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { + if r.CanGPU { + a.gpuUsed = false + } + if r.MultiThread() { + a.cpuUse -= wr.CPUs + } else { + a.cpuUse -= uint64(r.Threads) + } + + a.memUsedMin -= r.MinMemory + a.memUsedMax -= r.MaxMemory +} + +func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, res storiface.WorkerResources) bool { + + // TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running) + minNeedMem := res.MemReserved + a.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory + if minNeedMem > res.MemPhysical { + log.Debugf("sched: not scheduling on worker %d; not enough physical memory - need: %dM, have %dM", wid, minNeedMem/mib, res.MemPhysical/mib) + return false + } + + maxNeedMem := res.MemReserved + a.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory + + if maxNeedMem > res.MemSwap+res.MemPhysical { + log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib) + return false + } + + if needRes.MultiThread() { + if a.cpuUse > 0 { + log.Debugf("sched: not scheduling on worker %d; multicore process needs %d threads, %d in use, target %d", wid, res.CPUs, a.cpuUse, res.CPUs) + return false + } + } else { + if a.cpuUse+uint64(needRes.Threads) > res.CPUs { + log.Debugf("sched: not scheduling on worker %d; not enough threads, need %d, %d in use, target %d", wid, needRes.Threads, a.cpuUse, res.CPUs) + return false + } + } + + if len(res.GPUs) > 0 && needRes.CanGPU { + if a.gpuUsed { + log.Debugf("sched: not scheduling on worker %d; GPU in use", wid) + return false + } + } + + return true +} + +func (a *activeResources) utilization(wr storiface.WorkerResources) float64 { + var max float64 + + cpu := float64(a.cpuUse) / float64(wr.CPUs) + max = cpu + + memMin := float64(a.memUsedMin+wr.MemReserved) / float64(wr.MemPhysical) + if memMin > max { + max = memMin + } + + memMax := float64(a.memUsedMax+wr.MemReserved) / float64(wr.MemPhysical+wr.MemSwap) + if memMax > max { + max = memMax + } + + return max +} diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go new file mode 100644 index 000000000..6490e738e --- /dev/null +++ b/extern/sector-storage/sched_test.go @@ -0,0 +1,453 @@ +package sectorstorage + +import ( + "context" + "fmt" + "io" + "runtime" + "sync" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" + "github.com/filecoin-project/specs-storage/storage" +) + +func TestWithPriority(t *testing.T) { + ctx := context.Background() + + require.Equal(t, DefaultSchedPriority, getPriority(ctx)) + + ctx = WithPriority(ctx, 2222) + + require.Equal(t, 2222, getPriority(ctx)) +} + +type schedTestWorker struct { + name string + taskTypes map[sealtasks.TaskType]struct{} + paths []stores.StoragePath + + closed bool + closing chan struct{} +} + +func (s *schedTestWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + panic("implement me") +} + +func (s *schedTestWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storage.SectorCids, error) { + panic("implement me") +} + +func (s *schedTestWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { + panic("implement me") +} + +func (s *schedTestWorker) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) { + panic("implement me") +} + +func (s *schedTestWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { + panic("implement me") +} + +func (s *schedTestWorker) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { + panic("implement me") +} + +func (s *schedTestWorker) Remove(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (s *schedTestWorker) NewSector(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (s *schedTestWorker) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) { + panic("implement me") +} + +func (s *schedTestWorker) MoveStorage(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (s *schedTestWorker) Fetch(ctx context.Context, id abi.SectorID, ft stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error { + panic("implement me") +} + +func (s *schedTestWorker) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) error { + panic("implement me") +} + +func (s *schedTestWorker) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + panic("implement me") +} + +func (s *schedTestWorker) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) { + return s.taskTypes, nil +} + +func (s *schedTestWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { + return s.paths, nil +} + +func (s *schedTestWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) { + return storiface.WorkerInfo{ + Hostname: s.name, + Resources: storiface.WorkerResources{ + MemPhysical: 128 << 30, + MemSwap: 200 << 30, + MemReserved: 2 << 30, + CPUs: 32, + GPUs: []string{"a GPU"}, + }, + }, nil +} + +func (s *schedTestWorker) Closing(ctx context.Context) (<-chan struct{}, error) { + return s.closing, nil +} + +func (s *schedTestWorker) Close() error { + if !s.closed { + log.Info("close schedTestWorker") + s.closed = true + close(s.closing) + } + return nil +} + +var _ Worker = &schedTestWorker{} + +func addTestWorker(t *testing.T, sched *scheduler, index *stores.Index, name string, taskTypes map[sealtasks.TaskType]struct{}) { + w := &schedTestWorker{ + name: name, + taskTypes: taskTypes, + paths: []stores.StoragePath{{ID: "bb-8", Weight: 2, LocalPath: "food", CanSeal: true, CanStore: true}}, + + closing: make(chan struct{}), + } + + for _, path := range w.paths { + err := index.StorageAttach(context.TODO(), stores.StorageInfo{ + ID: path.ID, + URLs: nil, + Weight: path.Weight, + CanSeal: path.CanSeal, + CanStore: path.CanStore, + }, fsutil.FsStat{ + Capacity: 1 << 40, + Available: 1 << 40, + Reserved: 3, + }) + require.NoError(t, err) + } + + info, err := w.Info(context.TODO()) + require.NoError(t, err) + + sched.newWorkers <- &workerHandle{ + w: w, + wt: &workTracker{ + running: map[uint64]storiface.WorkerJob{}, + }, + info: info, + preparing: &activeResources{}, + active: &activeResources{}, + } +} + +func TestSchedStartStop(t *testing.T) { + spt := abi.RegisteredSealProof_StackedDrg32GiBV1 + sched := newScheduler(spt) + go sched.runSched() + + addTestWorker(t, sched, stores.NewIndex(), "fred", nil) + + require.NoError(t, sched.Close(context.TODO())) +} + +func TestSched(t *testing.T) { + ctx, done := context.WithTimeout(context.Background(), 30*time.Second) + defer done() + + spt := abi.RegisteredSealProof_StackedDrg32GiBV1 + + type workerSpec struct { + name string + taskTypes map[sealtasks.TaskType]struct{} + } + + noopAction := func(ctx context.Context, w Worker) error { + return nil + } + + type runMeta struct { + done map[string]chan struct{} + + wg sync.WaitGroup + } + + type task func(*testing.T, *scheduler, *stores.Index, *runMeta) + + sched := func(taskName, expectWorker string, sid abi.SectorNumber, taskType sealtasks.TaskType) task { + _, _, l, _ := runtime.Caller(1) + _, _, l2, _ := runtime.Caller(2) + + return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) { + done := make(chan struct{}) + rm.done[taskName] = done + + sel := newAllocSelector(index, stores.FTCache, stores.PathSealing) + + rm.wg.Add(1) + go func() { + defer rm.wg.Done() + + sectorNum := abi.SectorID{ + Miner: 8, + Number: sid, + } + + err := sched.Schedule(ctx, sectorNum, taskType, sel, func(ctx context.Context, w Worker) error { + wi, err := w.Info(ctx) + require.NoError(t, err) + + require.Equal(t, expectWorker, wi.Hostname) + + log.Info("IN ", taskName) + + for { + _, ok := <-done + if !ok { + break + } + } + + log.Info("OUT ", taskName) + + return nil + }, noopAction) + require.NoError(t, err, fmt.Sprint(l, l2)) + }() + + <-sched.testSync + } + } + + taskStarted := func(name string) task { + _, _, l, _ := runtime.Caller(1) + _, _, l2, _ := runtime.Caller(2) + return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) { + select { + case rm.done[name] <- struct{}{}: + case <-ctx.Done(): + t.Fatal("ctx error", ctx.Err(), l, l2) + } + } + } + + taskDone := func(name string) task { + _, _, l, _ := runtime.Caller(1) + _, _, l2, _ := runtime.Caller(2) + return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) { + select { + case rm.done[name] <- struct{}{}: + case <-ctx.Done(): + t.Fatal("ctx error", ctx.Err(), l, l2) + } + close(rm.done[name]) + } + } + + taskNotScheduled := func(name string) task { + _, _, l, _ := runtime.Caller(1) + _, _, l2, _ := runtime.Caller(2) + return func(t *testing.T, sched *scheduler, index *stores.Index, rm *runMeta) { + select { + case rm.done[name] <- struct{}{}: + t.Fatal("not expected", l, l2) + case <-time.After(10 * time.Millisecond): // TODO: better synchronization thingy + } + } + } + + testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { + return func(t *testing.T) { + index := stores.NewIndex() + + sched := newScheduler(spt) + sched.testSync = make(chan struct{}) + + go sched.runSched() + + for _, worker := range workers { + addTestWorker(t, sched, index, worker.name, worker.taskTypes) + } + + rm := runMeta{ + done: map[string]chan struct{}{}, + } + + for _, task := range tasks { + task(t, sched, index, &rm) + } + + log.Info("wait for async stuff") + rm.wg.Wait() + + require.NoError(t, sched.Close(context.TODO())) + } + } + + multTask := func(tasks ...task) task { + return func(t *testing.T, s *scheduler, index *stores.Index, meta *runMeta) { + for _, tsk := range tasks { + tsk(t, s, index, meta) + } + } + } + + t.Run("one-pc1", testFunc([]workerSpec{ + {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + }, []task{ + sched("pc1-1", "fred", 8, sealtasks.TTPreCommit1), + taskDone("pc1-1"), + })) + + t.Run("pc1-2workers-1", testFunc([]workerSpec{ + {name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, + {name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + }, []task{ + sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1), + taskDone("pc1-1"), + })) + + t.Run("pc1-2workers-2", testFunc([]workerSpec{ + {name: "fred1", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + {name: "fred2", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit2: {}}}, + }, []task{ + sched("pc1-1", "fred1", 8, sealtasks.TTPreCommit1), + taskDone("pc1-1"), + })) + + t.Run("pc1-block-pc2", testFunc([]workerSpec{ + {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + }, []task{ + sched("pc1", "fred", 8, sealtasks.TTPreCommit1), + taskStarted("pc1"), + + sched("pc2", "fred", 8, sealtasks.TTPreCommit2), + taskNotScheduled("pc2"), + + taskDone("pc1"), + taskDone("pc2"), + })) + + t.Run("pc2-block-pc1", testFunc([]workerSpec{ + {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + }, []task{ + sched("pc2", "fred", 8, sealtasks.TTPreCommit2), + taskStarted("pc2"), + + sched("pc1", "fred", 8, sealtasks.TTPreCommit1), + taskNotScheduled("pc1"), + + taskDone("pc2"), + taskDone("pc1"), + })) + + t.Run("pc1-batching", testFunc([]workerSpec{ + {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}}}, + }, []task{ + sched("t1", "fred", 8, sealtasks.TTPreCommit1), + taskStarted("t1"), + + sched("t2", "fred", 8, sealtasks.TTPreCommit1), + taskStarted("t2"), + + // with worker settings, we can only run 2 parallel PC1s + + // start 2 more to fill fetch buffer + + sched("t3", "fred", 8, sealtasks.TTPreCommit1), + taskNotScheduled("t3"), + + sched("t4", "fred", 8, sealtasks.TTPreCommit1), + taskNotScheduled("t4"), + + taskDone("t1"), + taskDone("t2"), + + taskStarted("t3"), + taskStarted("t4"), + + taskDone("t3"), + taskDone("t4"), + })) + + twoPC1 := func(prefix string, sid abi.SectorNumber, schedAssert func(name string) task) task { + return multTask( + sched(prefix+"-a", "fred", sid, sealtasks.TTPreCommit1), + schedAssert(prefix+"-a"), + + sched(prefix+"-b", "fred", sid+1, sealtasks.TTPreCommit1), + schedAssert(prefix+"-b"), + ) + } + + twoPC1Act := func(prefix string, schedAssert func(name string) task) task { + return multTask( + schedAssert(prefix+"-a"), + schedAssert(prefix+"-b"), + ) + } + + // run this one a bunch of times, it had a very annoying tendency to fail randomly + for i := 0; i < 40; i++ { + t.Run("pc1-pc2-prio", testFunc([]workerSpec{ + {name: "fred", taskTypes: map[sealtasks.TaskType]struct{}{sealtasks.TTPreCommit1: {}, sealtasks.TTPreCommit2: {}}}, + }, []task{ + // fill queues + twoPC1("w0", 0, taskStarted), + twoPC1("w1", 2, taskNotScheduled), + + // windowed + + sched("t1", "fred", 8, sealtasks.TTPreCommit1), + taskNotScheduled("t1"), + + sched("t2", "fred", 9, sealtasks.TTPreCommit1), + taskNotScheduled("t2"), + + sched("t3", "fred", 10, sealtasks.TTPreCommit2), + taskNotScheduled("t3"), + + twoPC1Act("w0", taskDone), + twoPC1Act("w1", taskStarted), + + twoPC1Act("w1", taskDone), + + taskStarted("t3"), + taskNotScheduled("t1"), + taskNotScheduled("t2"), + + taskDone("t3"), + + taskStarted("t1"), + taskStarted("t2"), + + taskDone("t1"), + taskDone("t2"), + })) + } +} diff --git a/extern/sector-storage/sched_watch.go b/extern/sector-storage/sched_watch.go new file mode 100644 index 000000000..d93cf1af3 --- /dev/null +++ b/extern/sector-storage/sched_watch.go @@ -0,0 +1,97 @@ +package sectorstorage + +import ( + "context" + "reflect" +) + +func (sh *scheduler) runWorkerWatcher() { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + nilch := reflect.ValueOf(new(chan struct{})).Elem() + + cases := []reflect.SelectCase{ + { + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(sh.closing), + }, + { + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(sh.watchClosing), + }, + } + + caseToWorker := map[int]WorkerID{} + + for { + n, rv, ok := reflect.Select(cases) + + switch { + case n == 0: // sh.closing + return + case n == 1: // sh.watchClosing + if !ok { + log.Errorf("watchClosing channel closed") + return + } + + wid, ok := rv.Interface().(WorkerID) + if !ok { + panic("got a non-WorkerID message") + } + + sh.workersLk.Lock() + workerClosing, err := sh.workers[wid].w.Closing(ctx) + sh.workersLk.Unlock() + if err != nil { + log.Errorf("getting worker closing channel: %+v", err) + select { + case sh.workerClosing <- wid: + case <-sh.closing: + return + } + + continue + } + + toSet := -1 + for i, sc := range cases { + if sc.Chan == nilch { + toSet = i + break + } + } + if toSet == -1 { + toSet = len(cases) + cases = append(cases, reflect.SelectCase{}) + } + + cases[toSet] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(workerClosing), + } + + caseToWorker[toSet] = wid + default: + wid, found := caseToWorker[n] + if !found { + log.Errorf("worker ID not found for case %d", n) + continue + } + + delete(caseToWorker, n) + cases[n] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: nilch, + } + + log.Warnf("worker %d dropped", wid) + select { + case sh.workerClosing <- wid: + case <-sh.closing: + return + } + } + } +} diff --git a/extern/sector-storage/sealtasks/task.go b/extern/sector-storage/sealtasks/task.go new file mode 100644 index 000000000..ad5ce01bb --- /dev/null +++ b/extern/sector-storage/sealtasks/task.go @@ -0,0 +1,57 @@ +package sealtasks + +type TaskType string + +const ( + TTAddPiece TaskType = "seal/v0/addpiece" + TTPreCommit1 TaskType = "seal/v0/precommit/1" + TTPreCommit2 TaskType = "seal/v0/precommit/2" + TTCommit1 TaskType = "seal/v0/commit/1" // NOTE: We use this to transfer the sector into miner-local storage for now; Don't use on workers! + TTCommit2 TaskType = "seal/v0/commit/2" + + TTFinalize TaskType = "seal/v0/finalize" + + TTFetch TaskType = "seal/v0/fetch" + TTUnseal TaskType = "seal/v0/unseal" + TTReadUnsealed TaskType = "seal/v0/unsealread" +) + +var order = map[TaskType]int{ + TTAddPiece: 7, + TTPreCommit1: 6, + TTPreCommit2: 5, + TTCommit2: 4, + TTCommit1: 3, + TTFetch: 2, + TTFinalize: 1, + TTUnseal: 0, + TTReadUnsealed: 0, +} + +var shortNames = map[TaskType]string{ + TTAddPiece: "AP ", + + TTPreCommit1: "PC1", + TTPreCommit2: "PC2", + TTCommit1: "C1 ", + TTCommit2: "C2 ", + + TTFinalize: "FIN", + + TTFetch: "GET", + TTUnseal: "UNS", + TTReadUnsealed: "RD ", +} + +func (a TaskType) Less(b TaskType) bool { + return order[a] < order[b] +} + +func (a TaskType) Short() string { + n, ok := shortNames[a] + if !ok { + return "UNK" + } + + return n +} diff --git a/extern/sector-storage/selector_alloc.go b/extern/sector-storage/selector_alloc.go new file mode 100644 index 000000000..cf7937587 --- /dev/null +++ b/extern/sector-storage/selector_alloc.go @@ -0,0 +1,65 @@ +package sectorstorage + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" +) + +type allocSelector struct { + index stores.SectorIndex + alloc stores.SectorFileType + ptype stores.PathType +} + +func newAllocSelector(index stores.SectorIndex, alloc stores.SectorFileType, ptype stores.PathType) *allocSelector { + return &allocSelector{ + index: index, + alloc: alloc, + ptype: ptype, + } +} + +func (s *allocSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, whnd *workerHandle) (bool, error) { + tasks, err := whnd.w.TaskTypes(ctx) + if err != nil { + return false, xerrors.Errorf("getting supported worker task types: %w", err) + } + if _, supported := tasks[task]; !supported { + return false, nil + } + + paths, err := whnd.w.Paths(ctx) + if err != nil { + return false, xerrors.Errorf("getting worker paths: %w", err) + } + + have := map[stores.ID]struct{}{} + for _, path := range paths { + have[path.ID] = struct{}{} + } + + best, err := s.index.StorageBestAlloc(ctx, s.alloc, spt, s.ptype) + if err != nil { + return false, xerrors.Errorf("finding best alloc storage: %w", err) + } + + for _, info := range best { + if _, ok := have[info.ID]; ok { + return true, nil + } + } + + return false, nil +} + +func (s *allocSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *workerHandle) (bool, error) { + return a.active.utilization(a.info.Resources) < b.active.utilization(b.info.Resources), nil +} + +var _ WorkerSelector = &allocSelector{} diff --git a/extern/sector-storage/selector_existing.go b/extern/sector-storage/selector_existing.go new file mode 100644 index 000000000..0a324ac5c --- /dev/null +++ b/extern/sector-storage/selector_existing.go @@ -0,0 +1,67 @@ +package sectorstorage + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" +) + +type existingSelector struct { + index stores.SectorIndex + sector abi.SectorID + alloc stores.SectorFileType + allowFetch bool +} + +func newExistingSelector(index stores.SectorIndex, sector abi.SectorID, alloc stores.SectorFileType, allowFetch bool) *existingSelector { + return &existingSelector{ + index: index, + sector: sector, + alloc: alloc, + allowFetch: allowFetch, + } +} + +func (s *existingSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, whnd *workerHandle) (bool, error) { + tasks, err := whnd.w.TaskTypes(ctx) + if err != nil { + return false, xerrors.Errorf("getting supported worker task types: %w", err) + } + if _, supported := tasks[task]; !supported { + return false, nil + } + + paths, err := whnd.w.Paths(ctx) + if err != nil { + return false, xerrors.Errorf("getting worker paths: %w", err) + } + + have := map[stores.ID]struct{}{} + for _, path := range paths { + have[path.ID] = struct{}{} + } + + best, err := s.index.StorageFindSector(ctx, s.sector, s.alloc, spt, s.allowFetch) + if err != nil { + return false, xerrors.Errorf("finding best storage: %w", err) + } + + for _, info := range best { + if _, ok := have[info.ID]; ok { + return true, nil + } + } + + return false, nil +} + +func (s *existingSelector) Cmp(ctx context.Context, task sealtasks.TaskType, a, b *workerHandle) (bool, error) { + return a.active.utilization(a.info.Resources) < b.active.utilization(b.info.Resources), nil +} + +var _ WorkerSelector = &existingSelector{} diff --git a/extern/sector-storage/selector_task.go b/extern/sector-storage/selector_task.go new file mode 100644 index 000000000..bf0788ef9 --- /dev/null +++ b/extern/sector-storage/selector_task.go @@ -0,0 +1,48 @@ +package sectorstorage + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" +) + +type taskSelector struct { + best []stores.StorageInfo //nolint: unused, structcheck +} + +func newTaskSelector() *taskSelector { + return &taskSelector{} +} + +func (s *taskSelector) Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, whnd *workerHandle) (bool, error) { + tasks, err := whnd.w.TaskTypes(ctx) + if err != nil { + return false, xerrors.Errorf("getting supported worker task types: %w", err) + } + _, supported := tasks[task] + + return supported, nil +} + +func (s *taskSelector) Cmp(ctx context.Context, _ sealtasks.TaskType, a, b *workerHandle) (bool, error) { + atasks, err := a.w.TaskTypes(ctx) + if err != nil { + return false, xerrors.Errorf("getting supported worker task types: %w", err) + } + btasks, err := b.w.TaskTypes(ctx) + if err != nil { + return false, xerrors.Errorf("getting supported worker task types: %w", err) + } + if len(atasks) != len(btasks) { + return len(atasks) < len(btasks), nil // prefer workers which can do less + } + + return a.active.utilization(a.info.Resources) < b.active.utilization(b.info.Resources), nil +} + +var _ WorkerSelector = &allocSelector{} diff --git a/extern/sector-storage/stats.go b/extern/sector-storage/stats.go new file mode 100644 index 000000000..ee88898a4 --- /dev/null +++ b/extern/sector-storage/stats.go @@ -0,0 +1,35 @@ +package sectorstorage + +import "github.com/filecoin-project/sector-storage/storiface" + +func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats { + m.sched.workersLk.Lock() + defer m.sched.workersLk.Unlock() + + out := map[uint64]storiface.WorkerStats{} + + for id, handle := range m.sched.workers { + out[uint64(id)] = storiface.WorkerStats{ + Info: handle.info, + MemUsedMin: handle.active.memUsedMin, + MemUsedMax: handle.active.memUsedMax, + GpuUsed: handle.active.gpuUsed, + CpuUse: handle.active.cpuUse, + } + } + + return out +} + +func (m *Manager) WorkerJobs() map[uint64][]storiface.WorkerJob { + m.sched.workersLk.Lock() + defer m.sched.workersLk.Unlock() + + out := map[uint64][]storiface.WorkerJob{} + + for id, handle := range m.sched.workers { + out[uint64(id)] = handle.wt.Running() + } + + return out +} diff --git a/extern/sector-storage/stores/filetype.go b/extern/sector-storage/stores/filetype.go new file mode 100644 index 000000000..650b92f71 --- /dev/null +++ b/extern/sector-storage/stores/filetype.go @@ -0,0 +1,140 @@ +package stores + +import ( + "fmt" + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +const ( + FTUnsealed SectorFileType = 1 << iota + FTSealed + FTCache + + FileTypes = iota +) + +const ( + FTNone SectorFileType = 0 +) + +const FSOverheadDen = 10 + +var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, + FTCache: 141, // 11 layers + D(2x ssize) + C + R +} + +var FsOverheadFinalized = map[SectorFileType]int{ + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, + FTCache: 2, +} + +type SectorFileType int + +func (t SectorFileType) String() string { + switch t { + case FTUnsealed: + return "unsealed" + case FTSealed: + return "sealed" + case FTCache: + return "cache" + default: + return fmt.Sprintf("", t) + } +} + +func (t SectorFileType) Has(singleType SectorFileType) bool { + return t&singleType == singleType +} + +func (t SectorFileType) SealSpaceUse(spt abi.RegisteredSealProof) (uint64, error) { + ssize, err := spt.SectorSize() + if err != nil { + return 0, xerrors.Errorf("getting sector size: %w", err) + } + + var need uint64 + for _, pathType := range PathTypes { + if !t.Has(pathType) { + continue + } + + oh, ok := FSOverheadSeal[pathType] + if !ok { + return 0, xerrors.Errorf("no seal overhead info for %s", pathType) + } + + need += uint64(oh) * uint64(ssize) / FSOverheadDen + } + + return need, nil +} + +func (t SectorFileType) All() [FileTypes]bool { + var out [FileTypes]bool + + for i := range out { + out[i] = t&(1< 0 + } + + return out +} + +type SectorPaths struct { + Id abi.SectorID + + Unsealed string + Sealed string + Cache string +} + +func ParseSectorID(baseName string) (abi.SectorID, error) { + var n abi.SectorNumber + var mid abi.ActorID + read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) + if err != nil { + return abi.SectorID{}, xerrors.Errorf("sscanf sector name ('%s'): %w", baseName, err) + } + + if read != 2 { + return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read) + } + + return abi.SectorID{ + Miner: mid, + Number: n, + }, nil +} + +func SectorName(sid abi.SectorID) string { + return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number) +} + +func PathByType(sps SectorPaths, fileType SectorFileType) string { + switch fileType { + case FTUnsealed: + return sps.Unsealed + case FTSealed: + return sps.Sealed + case FTCache: + return sps.Cache + } + + panic("requested unknown path type") +} + +func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) { + switch fileType { + case FTUnsealed: + sps.Unsealed = p + case FTSealed: + sps.Sealed = p + case FTCache: + sps.Cache = p + } +} diff --git a/extern/sector-storage/stores/http_handler.go b/extern/sector-storage/stores/http_handler.go new file mode 100644 index 000000000..4f0556138 --- /dev/null +++ b/extern/sector-storage/stores/http_handler.go @@ -0,0 +1,155 @@ +package stores + +import ( + "encoding/json" + "io" + "net/http" + "os" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/sector-storage/tarutil" +) + +var log = logging.Logger("stores") + +type FetchHandler struct { + *Local +} + +func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/ + mux := mux.NewRouter() + + mux.HandleFunc("/remote/stat/{id}", handler.remoteStatFs).Methods("GET") + mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") + mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") + + mux.ServeHTTP(w, r) +} + +func (handler *FetchHandler) remoteStatFs(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + id := ID(vars["id"]) + + st, err := handler.Local.FsStat(r.Context(), id) + switch err { + case errPathNotFound: + w.WriteHeader(404) + return + case nil: + break + default: + w.WriteHeader(500) + log.Errorf("%+v", err) + return + } + + if err := json.NewEncoder(w).Encode(&st); err != nil { + log.Warnf("error writing stat response: %+v", err) + } +} + +func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE GET %s", r.URL) + vars := mux.Vars(r) + + id, err := ParseSectorID(vars["id"]) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + // The caller has a lock on this sector already, no need to get one here + + // passing 0 spt because we don't allocate anything + paths, _, err := handler.Local.AcquireSector(r.Context(), id, 0, ft, FTNone, PathStorage, AcquireMove) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + // TODO: reserve local storage here + + path := PathByType(paths, ft) + if path == "" { + log.Error("acquired path was empty") + w.WriteHeader(500) + return + } + + stat, err := os.Stat(path) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + var rd io.Reader + if stat.IsDir() { + rd, err = tarutil.TarDirectory(path) + w.Header().Set("Content-Type", "application/x-tar") + } else { + rd, err = os.OpenFile(path, os.O_RDONLY, 0644) + w.Header().Set("Content-Type", "application/octet-stream") + } + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + w.WriteHeader(200) + if _, err := io.Copy(w, rd); err != nil { // TODO: default 32k buf may be too small + log.Error("%+v", err) + return + } +} + +func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE DELETE %s", r.URL) + vars := mux.Vars(r) + + id, err := ParseSectorID(vars["id"]) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + if err := handler.Remove(r.Context(), id, ft, false); err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } +} + +func ftFromString(t string) (SectorFileType, error) { + switch t { + case FTUnsealed.String(): + return FTUnsealed, nil + case FTSealed.String(): + return FTSealed, nil + case FTCache.String(): + return FTCache, nil + default: + return 0, xerrors.Errorf("unknown sector file type: '%s'", t) + } +} diff --git a/extern/sector-storage/stores/index.go b/extern/sector-storage/stores/index.go new file mode 100644 index 000000000..94858d3e8 --- /dev/null +++ b/extern/sector-storage/stores/index.go @@ -0,0 +1,442 @@ +package stores + +import ( + "context" + "github.com/filecoin-project/sector-storage/fsutil" + "net/url" + gopath "path" + "sort" + "sync" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" +) + +var HeartbeatInterval = 10 * time.Second +var SkippedHeartbeatThresh = HeartbeatInterval * 5 + +// ID identifies sector storage by UUID. One sector storage should map to one +// filesystem, local or networked / shared by multiple machines +type ID string + +type StorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + Weight uint64 + + CanSeal bool + CanStore bool +} + +type HealthReport struct { + Stat fsutil.FsStat + Err error +} + +type SectorStorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + Weight uint64 + + CanSeal bool + CanStore bool + + Primary bool +} + +type SectorIndex interface { // part of storage-miner api + StorageAttach(context.Context, StorageInfo, fsutil.FsStat) error + StorageInfo(context.Context, ID) (StorageInfo, error) + StorageReportHealth(context.Context, ID, HealthReport) error + + StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft SectorFileType, primary bool) error + StorageDropSector(ctx context.Context, storageId ID, s abi.SectorID, ft SectorFileType) error + StorageFindSector(ctx context.Context, sector abi.SectorID, ft SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]SectorStorageInfo, error) + + StorageBestAlloc(ctx context.Context, allocate SectorFileType, spt abi.RegisteredSealProof, pathType PathType) ([]StorageInfo, error) + + // atomically acquire locks on all sector file types. close ctx to unlock + StorageLock(ctx context.Context, sector abi.SectorID, read SectorFileType, write SectorFileType) error + StorageTryLock(ctx context.Context, sector abi.SectorID, read SectorFileType, write SectorFileType) (bool, error) +} + +type Decl struct { + abi.SectorID + SectorFileType +} + +type declMeta struct { + storage ID + primary bool +} + +type storageEntry struct { + info *StorageInfo + fsi fsutil.FsStat + + lastHeartbeat time.Time + heartbeatErr error +} + +type Index struct { + *indexLocks + lk sync.RWMutex + + sectors map[Decl][]*declMeta + stores map[ID]*storageEntry +} + +func NewIndex() *Index { + return &Index{ + indexLocks: &indexLocks{ + locks: map[abi.SectorID]*sectorLock{}, + }, + sectors: map[Decl][]*declMeta{}, + stores: map[ID]*storageEntry{}, + } +} + +func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + byID := map[ID]map[abi.SectorID]SectorFileType{} + + for id := range i.stores { + byID[id] = map[abi.SectorID]SectorFileType{} + } + for decl, ids := range i.sectors { + for _, id := range ids { + byID[id.storage][decl.SectorID] |= decl.SectorFileType + } + } + + out := map[ID][]Decl{} + for id, m := range byID { + out[id] = []Decl{} + for sectorID, fileType := range m { + out[id] = append(out[id], Decl{ + SectorID: sectorID, + SectorFileType: fileType, + }) + } + } + + return out, nil +} + +func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsStat) error { + i.lk.Lock() + defer i.lk.Unlock() + + log.Infof("New sector storage: %s", si.ID) + + if _, ok := i.stores[si.ID]; ok { + for _, u := range si.URLs { + if _, err := url.Parse(u); err != nil { + return xerrors.Errorf("failed to parse url %s: %w", si.URLs, err) + } + } + + uloop: + for _, u := range si.URLs { + for _, l := range i.stores[si.ID].info.URLs { + if u == l { + continue uloop + } + } + + i.stores[si.ID].info.URLs = append(i.stores[si.ID].info.URLs, u) + } + + return nil + } + i.stores[si.ID] = &storageEntry{ + info: &si, + fsi: st, + + lastHeartbeat: time.Now(), + } + return nil +} + +func (i *Index) StorageReportHealth(ctx context.Context, id ID, report HealthReport) error { + i.lk.Lock() + defer i.lk.Unlock() + + ent, ok := i.stores[id] + if !ok { + return xerrors.Errorf("health report for unknown storage: %s", id) + } + + ent.fsi = report.Stat + ent.heartbeatErr = report.Err + ent.lastHeartbeat = time.Now() + + return nil +} + +func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft SectorFileType, primary bool) error { + i.lk.Lock() + defer i.lk.Unlock() + +loop: + for _, fileType := range PathTypes { + if fileType&ft == 0 { + continue + } + + d := Decl{s, fileType} + + for _, sid := range i.sectors[d] { + if sid.storage == storageId { + if !sid.primary && primary { + sid.primary = true + } else { + log.Warnf("sector %v redeclared in %s", s, storageId) + } + continue loop + } + } + + i.sectors[d] = append(i.sectors[d], &declMeta{ + storage: storageId, + primary: primary, + }) + } + + return nil +} + +func (i *Index) StorageDropSector(ctx context.Context, storageId ID, s abi.SectorID, ft SectorFileType) error { + i.lk.Lock() + defer i.lk.Unlock() + + for _, fileType := range PathTypes { + if fileType&ft == 0 { + continue + } + + d := Decl{s, fileType} + + if len(i.sectors[d]) == 0 { + return nil + } + + rewritten := make([]*declMeta, 0, len(i.sectors[d])-1) + for _, sid := range i.sectors[d] { + if sid.storage == storageId { + continue + } + + rewritten = append(rewritten, sid) + } + if len(rewritten) == 0 { + delete(i.sectors, d) + return nil + } + + i.sectors[d] = rewritten + } + + return nil +} + +func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft SectorFileType, spt abi.RegisteredSealProof, allowFetch bool) ([]SectorStorageInfo, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + storageIDs := map[ID]uint64{} + isprimary := map[ID]bool{} + + for _, pathType := range PathTypes { + if ft&pathType == 0 { + continue + } + + for _, id := range i.sectors[Decl{s, pathType}] { + storageIDs[id.storage]++ + isprimary[id.storage] = isprimary[id.storage] || id.primary + } + } + + out := make([]SectorStorageInfo, 0, len(storageIDs)) + + for id, n := range storageIDs { + st, ok := i.stores[id] + if !ok { + log.Warnf("storage %s is not present in sector index (referenced by sector %v)", id, s) + continue + } + + urls := make([]string, len(st.info.URLs)) + for k, u := range st.info.URLs { + rl, err := url.Parse(u) + if err != nil { + return nil, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, ft.String(), SectorName(s)) + urls[k] = rl.String() + } + + out = append(out, SectorStorageInfo{ + ID: id, + URLs: urls, + Weight: st.info.Weight * n, // storage with more sector types is better + + CanSeal: st.info.CanSeal, + CanStore: st.info.CanStore, + + Primary: isprimary[id], + }) + } + + if allowFetch { + spaceReq, err := ft.SealSpaceUse(spt) + if err != nil { + return nil, xerrors.Errorf("estimating required space: %w", err) + } + + for id, st := range i.stores { + if !st.info.CanSeal { + continue + } + + if spaceReq > uint64(st.fsi.Available) { + log.Debugf("not selecting on %s, out of space (available: %d, need: %d)", st.info.ID, st.fsi.Available, spaceReq) + continue + } + + if time.Since(st.lastHeartbeat) > SkippedHeartbeatThresh { + log.Debugf("not selecting on %s, didn't receive heartbeats for %s", st.info.ID, time.Since(st.lastHeartbeat)) + continue + } + + if st.heartbeatErr != nil { + log.Debugf("not selecting on %s, heartbeat error: %s", st.info.ID, st.heartbeatErr) + continue + } + + if _, ok := storageIDs[id]; ok { + continue + } + + urls := make([]string, len(st.info.URLs)) + for k, u := range st.info.URLs { + rl, err := url.Parse(u) + if err != nil { + return nil, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, ft.String(), SectorName(s)) + urls[k] = rl.String() + } + + out = append(out, SectorStorageInfo{ + ID: id, + URLs: urls, + Weight: st.info.Weight * 0, // TODO: something better than just '0' + + CanSeal: st.info.CanSeal, + CanStore: st.info.CanStore, + + Primary: false, + }) + } + } + + return out, nil +} + +func (i *Index) StorageInfo(ctx context.Context, id ID) (StorageInfo, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + si, found := i.stores[id] + if !found { + return StorageInfo{}, xerrors.Errorf("sector store not found") + } + + return *si.info, nil +} + +func (i *Index) StorageBestAlloc(ctx context.Context, allocate SectorFileType, spt abi.RegisteredSealProof, pathType PathType) ([]StorageInfo, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + var candidates []storageEntry + + spaceReq, err := allocate.SealSpaceUse(spt) + if err != nil { + return nil, xerrors.Errorf("estimating required space: %w", err) + } + + for _, p := range i.stores { + if (pathType == PathSealing) && !p.info.CanSeal { + continue + } + if (pathType == PathStorage) && !p.info.CanStore { + continue + } + + if spaceReq > uint64(p.fsi.Available) { + log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq) + continue + } + + if time.Since(p.lastHeartbeat) > SkippedHeartbeatThresh { + log.Debugf("not allocating on %s, didn't receive heartbeats for %s", p.info.ID, time.Since(p.lastHeartbeat)) + continue + } + + if p.heartbeatErr != nil { + log.Debugf("not allocating on %s, heartbeat error: %s", p.info.ID, p.heartbeatErr) + continue + } + + candidates = append(candidates, *p) + } + + if len(candidates) == 0 { + return nil, xerrors.New("no good path found") + } + + sort.Slice(candidates, func(i, j int) bool { + iw := big.Mul(big.NewInt(int64(candidates[i].fsi.Available)), big.NewInt(int64(candidates[i].info.Weight))) + jw := big.Mul(big.NewInt(int64(candidates[j].fsi.Available)), big.NewInt(int64(candidates[j].info.Weight))) + + return iw.GreaterThan(jw) + }) + + out := make([]StorageInfo, len(candidates)) + for i, candidate := range candidates { + out[i] = *candidate.info + } + + return out, nil +} + +func (i *Index) FindSector(id abi.SectorID, typ SectorFileType) ([]ID, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + f, ok := i.sectors[Decl{ + SectorID: id, + SectorFileType: typ, + }] + if !ok { + return nil, nil + } + out := make([]ID, 0, len(f)) + for _, meta := range f { + out = append(out, meta.storage) + } + + return out, nil +} + +var _ SectorIndex = &Index{} diff --git a/extern/sector-storage/stores/index_locks.go b/extern/sector-storage/stores/index_locks.go new file mode 100644 index 000000000..8bf15b950 --- /dev/null +++ b/extern/sector-storage/stores/index_locks.go @@ -0,0 +1,154 @@ +package stores + +import ( + "context" + "sync" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type sectorLock struct { + cond *ctxCond + + r [FileTypes]uint + w SectorFileType + + refs uint // access with indexLocks.lk +} + +func (l *sectorLock) canLock(read SectorFileType, write SectorFileType) bool { + for i, b := range write.All() { + if b && l.r[i] > 0 { + return false + } + } + + // check that there are no locks taken for either read or write file types we want + return l.w&read == 0 && l.w&write == 0 +} + +func (l *sectorLock) tryLock(read SectorFileType, write SectorFileType) bool { + if !l.canLock(read, write) { + return false + } + + for i, set := range read.All() { + if set { + l.r[i]++ + } + } + + l.w |= write + + return true +} + +type lockFn func(l *sectorLock, ctx context.Context, read SectorFileType, write SectorFileType) (bool, error) + +func (l *sectorLock) tryLockSafe(ctx context.Context, read SectorFileType, write SectorFileType) (bool, error) { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + return l.tryLock(read, write), nil +} + +func (l *sectorLock) lock(ctx context.Context, read SectorFileType, write SectorFileType) (bool, error) { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + for !l.tryLock(read, write) { + if err := l.cond.Wait(ctx); err != nil { + return false, err + } + } + + return true, nil +} + +func (l *sectorLock) unlock(read SectorFileType, write SectorFileType) { + l.cond.L.Lock() + defer l.cond.L.Unlock() + + for i, set := range read.All() { + if set { + l.r[i]-- + } + } + + l.w &= ^write + + l.cond.Broadcast() +} + +type indexLocks struct { + lk sync.Mutex + + locks map[abi.SectorID]*sectorLock +} + +func (i *indexLocks) lockWith(ctx context.Context, lockFn lockFn, sector abi.SectorID, read SectorFileType, write SectorFileType) (bool, error) { + if read|write == 0 { + return false, nil + } + + if read|write > (1< %s: %w", sid, t, meta.ID, err) + } + } + } + + st.paths[meta.ID] = out + + return nil +} + +func (st *Local) open(ctx context.Context) error { + cfg, err := st.localStorage.GetStorage() + if err != nil { + return xerrors.Errorf("getting local storage config: %w", err) + } + + for _, path := range cfg.StoragePaths { + err := st.OpenPath(ctx, path.Path) + if err != nil { + return xerrors.Errorf("opening path %s: %w", path.Path, err) + } + } + + go st.reportHealth(ctx) + + return nil +} + +func (st *Local) reportHealth(ctx context.Context) { + // randomize interval by ~10% + interval := (HeartbeatInterval*100_000 + time.Duration(rand.Int63n(10_000))) / 100_000 + + for { + select { + case <-time.After(interval): + case <-ctx.Done(): + return + } + + st.localLk.RLock() + + toReport := map[ID]HealthReport{} + for id, p := range st.paths { + stat, err := p.stat(st.localStorage) + + toReport[id] = HealthReport{ + Stat: stat, + Err: err, + } + } + + st.localLk.RUnlock() + + for id, report := range toReport { + if err := st.index.StorageReportHealth(ctx, id, report); err != nil { + log.Warnf("error reporting storage health for %s: %+v", id, report) + } + } + } +} + +func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, ft SectorFileType, storageIDs SectorPaths, overheadTab map[SectorFileType]int) (func(), error) { + ssize, err := spt.SectorSize() + if err != nil { + return nil, xerrors.Errorf("getting sector size: %w", err) + } + + st.localLk.Lock() + + done := func() {} + deferredDone := func() { done() } + defer func() { + st.localLk.Unlock() + deferredDone() + }() + + for _, fileType := range PathTypes { + if fileType&ft == 0 { + continue + } + + id := ID(PathByType(storageIDs, fileType)) + + p, ok := st.paths[id] + if !ok { + return nil, errPathNotFound + } + + stat, err := p.stat(st.localStorage) + if err != nil { + return nil, xerrors.Errorf("getting local storage stat: %w", err) + } + + overhead := int64(overheadTab[fileType]) * int64(ssize) / FSOverheadDen + + if stat.Available < overhead { + return nil, xerrors.Errorf("can't reserve %d bytes in '%s' (id:%s), only %d available", overhead, p.local, id, stat.Available) + } + + p.reserved += overhead + + prevDone := done + done = func() { + prevDone() + + st.localLk.Lock() + defer st.localLk.Unlock() + + p.reserved -= overhead + } + } + + deferredDone = func() {} + return done, nil +} + +func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { + if existing|allocate != existing^allocate { + return SectorPaths{}, SectorPaths{}, xerrors.New("can't both find and allocate a sector") + } + + st.localLk.RLock() + defer st.localLk.RUnlock() + + var out SectorPaths + var storageIDs SectorPaths + + for _, fileType := range PathTypes { + if fileType&existing == 0 { + continue + } + + si, err := st.index.StorageFindSector(ctx, sid, fileType, spt, false) + if err != nil { + log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err) + continue + } + + for _, info := range si { + p, ok := st.paths[info.ID] + if !ok { + continue + } + + if p.local == "" { // TODO: can that even be the case? + continue + } + + spath := p.sectorPath(sid, fileType) + SetPathByType(&out, fileType, spath) + SetPathByType(&storageIDs, fileType, string(info.ID)) + + existing ^= fileType + break + } + } + + for _, fileType := range PathTypes { + if fileType&allocate == 0 { + continue + } + + sis, err := st.index.StorageBestAlloc(ctx, fileType, spt, pathType) + if err != nil { + return SectorPaths{}, SectorPaths{}, xerrors.Errorf("finding best storage for allocating : %w", err) + } + + var best string + var bestID ID + + for _, si := range sis { + p, ok := st.paths[si.ID] + if !ok { + continue + } + + if p.local == "" { // TODO: can that even be the case? + continue + } + + if (pathType == PathSealing) && !si.CanSeal { + continue + } + + if (pathType == PathStorage) && !si.CanStore { + continue + } + + // TODO: Check free space + + best = p.sectorPath(sid, fileType) + bestID = si.ID + break + } + + if best == "" { + return SectorPaths{}, SectorPaths{}, xerrors.Errorf("couldn't find a suitable path for a sector") + } + + SetPathByType(&out, fileType, best) + SetPathByType(&storageIDs, fileType, string(bestID)) + allocate ^= fileType + } + + return out, storageIDs, nil +} + +func (st *Local) Local(ctx context.Context) ([]StoragePath, error) { + st.localLk.RLock() + defer st.localLk.RUnlock() + + var out []StoragePath + for id, p := range st.paths { + if p.local == "" { + continue + } + + si, err := st.index.StorageInfo(ctx, id) + if err != nil { + return nil, xerrors.Errorf("get storage info for %s: %w", id, err) + } + + out = append(out, StoragePath{ + ID: id, + Weight: si.Weight, + LocalPath: p.local, + CanSeal: si.CanSeal, + CanStore: si.CanStore, + }) + } + + return out, nil +} + +func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ SectorFileType, force bool) error { + if bits.OnesCount(uint(typ)) != 1 { + return xerrors.New("delete expects one file type") + } + + si, err := st.index.StorageFindSector(ctx, sid, typ, 0, false) + if err != nil { + return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) + } + + if len(si) == 0 && !force { + return xerrors.Errorf("can't delete sector %v(%d), not found", sid, typ) + } + + for _, info := range si { + if err := st.removeSector(ctx, sid, typ, info.ID); err != nil { + return err + } + } + + return nil +} + +func (st *Local) RemoveCopies(ctx context.Context, sid abi.SectorID, typ SectorFileType) error { + if bits.OnesCount(uint(typ)) != 1 { + return xerrors.New("delete expects one file type") + } + + si, err := st.index.StorageFindSector(ctx, sid, typ, 0, false) + if err != nil { + return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) + } + + var hasPrimary bool + for _, info := range si { + if info.Primary { + hasPrimary = true + break + } + } + + if !hasPrimary { + log.Warnf("RemoveCopies: no primary copies of sector %v (%s), not removing anything", sid, typ) + return nil + } + + for _, info := range si { + if info.Primary { + continue + } + + if err := st.removeSector(ctx, sid, typ, info.ID); err != nil { + return err + } + } + + return nil +} + +func (st *Local) removeSector(ctx context.Context, sid abi.SectorID, typ SectorFileType, storage ID) error { + p, ok := st.paths[storage] + if !ok { + return nil + } + + if p.local == "" { // TODO: can that even be the case? + return nil + } + + if err := st.index.StorageDropSector(ctx, storage, sid, typ); err != nil { + return xerrors.Errorf("dropping sector from index: %w", err) + } + + spath := p.sectorPath(sid, typ) + log.Infof("remove %s", spath) + + if err := os.RemoveAll(spath); err != nil { + log.Errorf("removing sector (%v) from %s: %+v", sid, spath, err) + } + + return nil +} + +func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error { + dest, destIds, err := st.AcquireSector(ctx, s, spt, FTNone, types, PathStorage, AcquireMove) + if err != nil { + return xerrors.Errorf("acquire dest storage: %w", err) + } + + src, srcIds, err := st.AcquireSector(ctx, s, spt, types, FTNone, PathStorage, AcquireMove) + if err != nil { + return xerrors.Errorf("acquire src storage: %w", err) + } + + for _, fileType := range PathTypes { + if fileType&types == 0 { + continue + } + + sst, err := st.index.StorageInfo(ctx, ID(PathByType(srcIds, fileType))) + if err != nil { + return xerrors.Errorf("failed to get source storage info: %w", err) + } + + dst, err := st.index.StorageInfo(ctx, ID(PathByType(destIds, fileType))) + if err != nil { + return xerrors.Errorf("failed to get source storage info: %w", err) + } + + if sst.ID == dst.ID { + log.Debugf("not moving %v(%d); src and dest are the same", s, fileType) + continue + } + + if sst.CanStore { + log.Debugf("not moving %v(%d); source supports storage", s, fileType) + continue + } + + log.Debugf("moving %v(%d) to storage: %s(se:%t; st:%t) -> %s(se:%t; st:%t)", s, fileType, sst.ID, sst.CanSeal, sst.CanStore, dst.ID, dst.CanSeal, dst.CanStore) + + if err := st.index.StorageDropSector(ctx, ID(PathByType(srcIds, fileType)), s, fileType); err != nil { + return xerrors.Errorf("dropping source sector from index: %w", err) + } + + if err := move(PathByType(src, fileType), PathByType(dest, fileType)); err != nil { + // TODO: attempt some recovery (check if src is still there, re-declare) + return xerrors.Errorf("moving sector %v(%d): %w", s, fileType, err) + } + + if err := st.index.StorageDeclareSector(ctx, ID(PathByType(destIds, fileType)), s, fileType, true); err != nil { + return xerrors.Errorf("declare sector %d(t:%d) -> %s: %w", s, fileType, ID(PathByType(destIds, fileType)), err) + } + } + + return nil +} + +var errPathNotFound = xerrors.Errorf("fsstat: path not found") + +func (st *Local) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) { + st.localLk.RLock() + defer st.localLk.RUnlock() + + p, ok := st.paths[id] + if !ok { + return fsutil.FsStat{}, errPathNotFound + } + + return p.stat(st.localStorage) +} + +var _ Store = &Local{} diff --git a/extern/sector-storage/stores/local_test.go b/extern/sector-storage/stores/local_test.go new file mode 100644 index 000000000..56ac7c020 --- /dev/null +++ b/extern/sector-storage/stores/local_test.go @@ -0,0 +1,94 @@ +package stores + +import ( + "context" + "encoding/json" + "github.com/filecoin-project/sector-storage/fsutil" + "github.com/google/uuid" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +const pathSize = 16 << 20 + +type TestingLocalStorage struct { + root string + c StorageConfig +} + +func (t *TestingLocalStorage) DiskUsage(path string) (int64, error) { + return 1, nil +} + +func (t *TestingLocalStorage) GetStorage() (StorageConfig, error) { + return t.c, nil +} + +func (t *TestingLocalStorage) SetStorage(f func(*StorageConfig)) error { + f(&t.c) + return nil +} + +func (t *TestingLocalStorage) Stat(path string) (fsutil.FsStat, error) { + return fsutil.FsStat{ + Capacity: pathSize, + Available: pathSize, + }, nil +} + +func (t *TestingLocalStorage) init(subpath string) error { + path := filepath.Join(t.root, subpath) + if err := os.Mkdir(path, 0755); err != nil { + return err + } + + metaFile := filepath.Join(path, MetaFile) + + meta := &LocalStorageMeta{ + ID: ID(uuid.New().String()), + Weight: 1, + CanSeal: true, + CanStore: true, + } + + mb, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + + if err := ioutil.WriteFile(metaFile, mb, 0644); err != nil { + return err + } + + return nil +} + +var _ LocalStorage = &TestingLocalStorage{} + +func TestLocalStorage(t *testing.T) { + ctx := context.TODO() + + root, err := ioutil.TempDir("", "sector-storage-teststorage-") + require.NoError(t, err) + + tstor := &TestingLocalStorage{ + root: root, + } + + index := NewIndex() + + st, err := NewLocal(ctx, tstor, index, nil) + require.NoError(t, err) + + p1 := "1" + require.NoError(t, tstor.init("1")) + + err = st.OpenPath(ctx, filepath.Join(tstor.root, p1)) + require.NoError(t, err) + + // TODO: put more things here +} diff --git a/extern/sector-storage/stores/remote.go b/extern/sector-storage/stores/remote.go new file mode 100644 index 000000000..12587a86e --- /dev/null +++ b/extern/sector-storage/stores/remote.go @@ -0,0 +1,406 @@ +package stores + +import ( + "context" + "encoding/json" + "github.com/filecoin-project/sector-storage/fsutil" + "io/ioutil" + "math/bits" + "mime" + "net/http" + "net/url" + "os" + gopath "path" + "path/filepath" + "sort" + "sync" + + "github.com/hashicorp/go-multierror" + files "github.com/ipfs/go-ipfs-files" + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/sector-storage/storiface" + "github.com/filecoin-project/sector-storage/tarutil" +) + +var FetchTempSubdir = "fetching" + +type Remote struct { + local *Local + index SectorIndex + auth http.Header + + limit chan struct{} + + fetchLk sync.Mutex + fetching map[abi.SectorID]chan struct{} +} + +func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types SectorFileType) error { + // TODO: do this on remotes too + // (not that we really need to do that since it's always called by the + // worker which pulled the copy) + + return r.local.RemoveCopies(ctx, s, types) +} + +func NewRemote(local *Local, index SectorIndex, auth http.Header, fetchLimit int) *Remote { + return &Remote{ + local: local, + index: index, + auth: auth, + + limit: make(chan struct{}, fetchLimit), + + fetching: map[abi.SectorID]chan struct{}{}, + } +} + +func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { + if existing|allocate != existing^allocate { + return SectorPaths{}, SectorPaths{}, xerrors.New("can't both find and allocate a sector") + } + + for { + r.fetchLk.Lock() + + c, locked := r.fetching[s] + if !locked { + r.fetching[s] = make(chan struct{}) + r.fetchLk.Unlock() + break + } + + r.fetchLk.Unlock() + + select { + case <-c: + continue + case <-ctx.Done(): + return SectorPaths{}, SectorPaths{}, ctx.Err() + } + } + + defer func() { + r.fetchLk.Lock() + close(r.fetching[s]) + delete(r.fetching, s) + r.fetchLk.Unlock() + }() + + paths, stores, err := r.local.AcquireSector(ctx, s, spt, existing, allocate, pathType, op) + if err != nil { + return SectorPaths{}, SectorPaths{}, xerrors.Errorf("local acquire error: %w", err) + } + + var toFetch SectorFileType + for _, fileType := range PathTypes { + if fileType&existing == 0 { + continue + } + + if PathByType(paths, fileType) == "" { + toFetch |= fileType + } + } + + apaths, ids, err := r.local.AcquireSector(ctx, s, spt, FTNone, toFetch, pathType, op) + if err != nil { + return SectorPaths{}, SectorPaths{}, xerrors.Errorf("allocate local sector for fetching: %w", err) + } + + odt := FSOverheadSeal + if pathType == PathStorage { + odt = FsOverheadFinalized + } + + releaseStorage, err := r.local.Reserve(ctx, s, spt, toFetch, ids, odt) + if err != nil { + return SectorPaths{}, SectorPaths{}, xerrors.Errorf("reserving storage space: %w", err) + } + defer releaseStorage() + + for _, fileType := range PathTypes { + if fileType&existing == 0 { + continue + } + + if PathByType(paths, fileType) != "" { + continue + } + + dest := PathByType(apaths, fileType) + storageID := PathByType(ids, fileType) + + url, err := r.acquireFromRemote(ctx, s, fileType, dest) + if err != nil { + return SectorPaths{}, SectorPaths{}, err + } + + SetPathByType(&paths, fileType, dest) + SetPathByType(&stores, fileType, storageID) + + if err := r.index.StorageDeclareSector(ctx, ID(storageID), s, fileType, op == AcquireMove); err != nil { + log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) + continue + } + + if op == AcquireMove { + if err := r.deleteFromRemote(ctx, url); err != nil { + log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err) + } + } + } + + return paths, stores, nil +} + +func tempFetchDest(spath string, create bool) (string, error) { + st, b := filepath.Split(spath) + tempdir := filepath.Join(st, FetchTempSubdir) + if create { + if err := os.MkdirAll(tempdir, 0755); err != nil { + return "", xerrors.Errorf("creating temp fetch dir: %w", err) + } + } + + return filepath.Join(tempdir, b), nil +} + +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType SectorFileType, dest string) (string, error) { + si, err := r.index.StorageFindSector(ctx, s, fileType, 0, false) + if err != nil { + return "", err + } + + if len(si) == 0 { + return "", xerrors.Errorf("failed to acquire sector %v from remote(%d): %w", s, fileType, storiface.ErrSectorNotFound) + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Weight < si[j].Weight + }) + + var merr error + for _, info := range si { + // TODO: see what we have local, prefer that + + for _, url := range info.URLs { + tempDest, err := tempFetchDest(dest, true) + if err != nil { + return "", err + } + + if err := os.RemoveAll(dest); err != nil { + return "", xerrors.Errorf("removing dest: %w", err) + } + + err = r.fetch(ctx, url, tempDest) + if err != nil { + merr = multierror.Append(merr, xerrors.Errorf("fetch error %s (storage %s) -> %s: %w", url, info.ID, tempDest, err)) + continue + } + + if err := move(tempDest, dest); err != nil { + return "", xerrors.Errorf("fetch move error (storage %s) %s -> %s: %w", info.ID, tempDest, dest, err) + } + + if merr != nil { + log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) + } + return url, nil + } + } + + return "", xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) +} + +func (r *Remote) fetch(ctx context.Context, url, outname string) error { + log.Infof("Fetch %s -> %s", url, outname) + + if len(r.limit) >= cap(r.limit) { + log.Infof("Throttling fetch, %d already running", len(r.limit)) + } + + // TODO: Smarter throttling + // * Priority (just going sequentially is still pretty good) + // * Per interface + // * Aware of remote load + select { + case r.limit <- struct{}{}: + defer func() { <-r.limit }() + case <-ctx.Done(): + return xerrors.Errorf("context error while waiting for fetch limiter: %w", ctx.Err()) + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + /*bar := pb.New64(w.sizeForType(typ)) + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + barreader := bar.NewProxyReader(resp.Body) + + bar.Start() + defer bar.Finish()*/ + + mediatype, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return xerrors.Errorf("parse media type: %w", err) + } + + if err := os.RemoveAll(outname); err != nil { + return xerrors.Errorf("removing dest: %w", err) + } + + switch mediatype { + case "application/x-tar": + return tarutil.ExtractTar(resp.Body, outname) + case "application/octet-stream": + return files.WriteTo(files.NewReaderFile(resp.Body), outname) + default: + return xerrors.Errorf("unknown content type: '%s'", mediatype) + } +} + +func (r *Remote) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error { + // Make sure we have the data local + _, _, err := r.AcquireSector(ctx, s, spt, types, FTNone, PathStorage, AcquireMove) + if err != nil { + return xerrors.Errorf("acquire src storage (remote): %w", err) + } + + return r.local.MoveStorage(ctx, s, spt, types) +} + +func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ SectorFileType, force bool) error { + if bits.OnesCount(uint(typ)) != 1 { + return xerrors.New("delete expects one file type") + } + + if err := r.local.Remove(ctx, sid, typ, force); err != nil { + return xerrors.Errorf("remove from local: %w", err) + } + + si, err := r.index.StorageFindSector(ctx, sid, typ, 0, false) + if err != nil { + return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) + } + + for _, info := range si { + for _, url := range info.URLs { + if err := r.deleteFromRemote(ctx, url); err != nil { + log.Warnf("remove %s: %+v", url, err) + continue + } + break + } + } + + return nil +} + +func (r *Remote) deleteFromRemote(ctx context.Context, url string) error { + log.Infof("Delete %s", url) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + return nil +} + +func (r *Remote) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) { + st, err := r.local.FsStat(ctx, id) + switch err { + case nil: + return st, nil + case errPathNotFound: + break + default: + return fsutil.FsStat{}, xerrors.Errorf("local stat: %w", err) + } + + si, err := r.index.StorageInfo(ctx, id) + if err != nil { + return fsutil.FsStat{}, xerrors.Errorf("getting remote storage info: %w", err) + } + + if len(si.URLs) == 0 { + return fsutil.FsStat{}, xerrors.Errorf("no known URLs for remote storage %s", id) + } + + rl, err := url.Parse(si.URLs[0]) + if err != nil { + return fsutil.FsStat{}, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, "stat", string(id)) + + req, err := http.NewRequest("GET", rl.String(), nil) + if err != nil { + return fsutil.FsStat{}, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fsutil.FsStat{}, xerrors.Errorf("do request: %w", err) + } + switch resp.StatusCode { + case 200: + break + case 404: + return fsutil.FsStat{}, errPathNotFound + case 500: + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fsutil.FsStat{}, xerrors.Errorf("fsstat: got http 500, then failed to read the error: %w", err) + } + + return fsutil.FsStat{}, xerrors.Errorf("fsstat: got http 500: %s", string(b)) + } + + var out fsutil.FsStat + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return fsutil.FsStat{}, xerrors.Errorf("decoding fsstat: %w", err) + } + + defer resp.Body.Close() + + return out, nil +} + +var _ Store = &Remote{} diff --git a/extern/sector-storage/stores/util_unix.go b/extern/sector-storage/stores/util_unix.go new file mode 100644 index 000000000..eeb691ddf --- /dev/null +++ b/extern/sector-storage/stores/util_unix.go @@ -0,0 +1,43 @@ +package stores + +import ( + "bytes" + "os/exec" + "path/filepath" + "strings" + + "github.com/mitchellh/go-homedir" + "golang.org/x/xerrors" +) + +func move(from, to string) error { + from, err := homedir.Expand(from) + if err != nil { + return xerrors.Errorf("move: expanding from: %w", err) + } + + to, err = homedir.Expand(to) + if err != nil { + return xerrors.Errorf("move: expanding to: %w", err) + } + + if filepath.Base(from) != filepath.Base(to) { + return xerrors.Errorf("move: base names must match ('%s' != '%s')", filepath.Base(from), filepath.Base(to)) + } + + log.Debugw("move sector data", "from", from, "to", to) + + toDir := filepath.Dir(to) + + // `mv` has decades of experience in moving files quickly; don't pretend we + // can do better + + var errOut bytes.Buffer + cmd := exec.Command("/usr/bin/env", "mv", "-t", toDir, from) + cmd.Stderr = &errOut + if err := cmd.Run(); err != nil { + return xerrors.Errorf("exec mv (stderr: %s): %w", strings.TrimSpace(errOut.String()), err) + } + + return nil +} diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go new file mode 100644 index 000000000..6e16018f0 --- /dev/null +++ b/extern/sector-storage/storiface/ffi.go @@ -0,0 +1,17 @@ +package storiface + +import ( + "errors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +var ErrSectorNotFound = errors.New("sector not found") + +type UnpaddedByteIndex uint64 + +func (i UnpaddedByteIndex) Padded() PaddedByteIndex { + return PaddedByteIndex(abi.UnpaddedPieceSize(i).Padded()) +} + +type PaddedByteIndex uint64 diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go new file mode 100644 index 000000000..01ef59d36 --- /dev/null +++ b/extern/sector-storage/storiface/worker.go @@ -0,0 +1,41 @@ +package storiface + +import ( + "time" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type WorkerInfo struct { + Hostname string + + Resources WorkerResources +} + +type WorkerResources struct { + MemPhysical uint64 + MemSwap uint64 + + MemReserved uint64 // Used by system / other processes + + CPUs uint64 // Logical cores + GPUs []string +} + +type WorkerStats struct { + Info WorkerInfo + + MemUsedMin uint64 + MemUsedMax uint64 + GpuUsed bool + CpuUse uint64 +} + +type WorkerJob struct { + ID uint64 + Sector abi.SectorID + Task sealtasks.TaskType + + Start time.Time +} diff --git a/extern/sector-storage/tarutil/systar.go b/extern/sector-storage/tarutil/systar.go new file mode 100644 index 000000000..94de58ea8 --- /dev/null +++ b/extern/sector-storage/tarutil/systar.go @@ -0,0 +1,92 @@ +package tarutil + +import ( + "archive/tar" + "golang.org/x/xerrors" + "io" + "io/ioutil" + "os" + "path/filepath" + + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("tarutil") // nolint + +func ExtractTar(body io.Reader, dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return xerrors.Errorf("mkdir: %w", err) + } + + tr := tar.NewReader(body) + for { + header, err := tr.Next() + switch err { + default: + return err + case io.EOF: + return nil + + case nil: + } + + f, err := os.Create(filepath.Join(dir, header.Name)) + if err != nil { + return xerrors.Errorf("creating file %s: %w", filepath.Join(dir, header.Name), err) + } + + if _, err := io.Copy(f, tr); err != nil { + return err + } + + if err := f.Close(); err != nil { + return err + } + } +} + +func TarDirectory(dir string) (io.ReadCloser, error) { + r, w := io.Pipe() + + go func() { + _ = w.CloseWithError(writeTarDirectory(dir, w)) + }() + + return r, nil +} + +func writeTarDirectory(dir string, w io.Writer) error { + tw := tar.NewWriter(w) + + files, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + + for _, file := range files { + h, err := tar.FileInfoHeader(file, "") + if err != nil { + return xerrors.Errorf("getting header for file %s: %w", file.Name(), err) + } + + if err := tw.WriteHeader(h); err != nil { + return xerrors.Errorf("wiritng header for file %s: %w", file.Name(), err) + } + + f, err := os.OpenFile(filepath.Join(dir, file.Name()), os.O_RDONLY, 644) // nolint + if err != nil { + return xerrors.Errorf("opening %s for reading: %w", file.Name(), err) + } + + if _, err := io.Copy(tw, f); err != nil { + return xerrors.Errorf("copy data for file %s: %w", file.Name(), err) + } + + if err := f.Close(); err != nil { + return err + } + + } + + return nil +} diff --git a/extern/sector-storage/testworker_test.go b/extern/sector-storage/testworker_test.go new file mode 100644 index 000000000..40151a84d --- /dev/null +++ b/extern/sector-storage/testworker_test.go @@ -0,0 +1,127 @@ +package sectorstorage + +import ( + "context" + "io" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/mock" + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +type testWorker struct { + acceptTasks map[sealtasks.TaskType]struct{} + lstor *stores.Local + + mockSeal *mock.SectorMgr +} + +func newTestWorker(wcfg WorkerConfig, lstor *stores.Local) *testWorker { + ssize, err := wcfg.SealProof.SectorSize() + if err != nil { + panic(err) + } + + acceptTasks := map[sealtasks.TaskType]struct{}{} + for _, taskType := range wcfg.TaskTypes { + acceptTasks[taskType] = struct{}{} + } + + return &testWorker{ + acceptTasks: acceptTasks, + lstor: lstor, + + mockSeal: mock.NewMockSectorMgr(ssize, nil), + } +} + +func (t *testWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + return t.mockSeal.SealPreCommit1(ctx, sector, ticket, pieces) +} + +func (t *testWorker) NewSector(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (t *testWorker) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) error { + panic("implement me") +} + +func (t *testWorker) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + panic("implement me") +} + +func (t *testWorker) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) { + return t.mockSeal.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData) +} + +func (t *testWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storage.SectorCids, error) { + panic("implement me") +} + +func (t *testWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { + panic("implement me") +} + +func (t *testWorker) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) { + panic("implement me") +} + +func (t *testWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { + panic("implement me") +} + +func (t *testWorker) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { + panic("implement me") +} + +func (t *testWorker) Remove(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (t *testWorker) MoveStorage(ctx context.Context, sector abi.SectorID) error { + panic("implement me") +} + +func (t *testWorker) Fetch(ctx context.Context, id abi.SectorID, fileType stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error { + return nil +} + +func (t *testWorker) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) { + return t.acceptTasks, nil +} + +func (t *testWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { + return t.lstor.Local(ctx) +} + +func (t *testWorker) Info(ctx context.Context) (storiface.WorkerInfo, error) { + res := ResourceTable[sealtasks.TTPreCommit2][abi.RegisteredSealProof_StackedDrg2KiBV1] + + return storiface.WorkerInfo{ + Hostname: "testworkerer", + Resources: storiface.WorkerResources{ + MemPhysical: res.MinMemory * 3, + MemSwap: 0, + MemReserved: res.MinMemory, + CPUs: 32, + GPUs: nil, + }, + }, nil +} + +func (t *testWorker) Closing(ctx context.Context) (<-chan struct{}, error) { + return ctx.Done(), nil +} + +func (t *testWorker) Close() error { + panic("implement me") +} + +var _ Worker = &testWorker{} diff --git a/extern/sector-storage/work_tracker.go b/extern/sector-storage/work_tracker.go new file mode 100644 index 000000000..7453752c9 --- /dev/null +++ b/extern/sector-storage/work_tracker.go @@ -0,0 +1,129 @@ +package sectorstorage + +import ( + "context" + "io" + "sync" + "time" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/sector-storage/sealtasks" + "github.com/filecoin-project/sector-storage/stores" + "github.com/filecoin-project/sector-storage/storiface" +) + +type workTracker struct { + lk sync.Mutex + + ctr uint64 + running map[uint64]storiface.WorkerJob + + // TODO: done, aggregate stats, queue stats, scheduler feedback +} + +func (wt *workTracker) track(sid abi.SectorID, task sealtasks.TaskType) func() { + wt.lk.Lock() + defer wt.lk.Unlock() + + id := wt.ctr + wt.ctr++ + + wt.running[id] = storiface.WorkerJob{ + ID: id, + Sector: sid, + Task: task, + Start: time.Now(), + } + + return func() { + wt.lk.Lock() + defer wt.lk.Unlock() + + delete(wt.running, id) + } +} + +func (wt *workTracker) worker(w Worker) Worker { + return &trackedWorker{ + Worker: w, + tracker: wt, + } +} + +func (wt *workTracker) Running() []storiface.WorkerJob { + wt.lk.Lock() + defer wt.lk.Unlock() + + out := make([]storiface.WorkerJob, 0, len(wt.running)) + for _, job := range wt.running { + out = append(out, job) + } + + return out +} + +type trackedWorker struct { + Worker + + tracker *workTracker +} + +func (t *trackedWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + defer t.tracker.track(sector, sealtasks.TTPreCommit1)() + + return t.Worker.SealPreCommit1(ctx, sector, ticket, pieces) +} + +func (t *trackedWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, pc1o storage.PreCommit1Out) (storage.SectorCids, error) { + defer t.tracker.track(sector, sealtasks.TTPreCommit2)() + + return t.Worker.SealPreCommit2(ctx, sector, pc1o) +} + +func (t *trackedWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { + defer t.tracker.track(sector, sealtasks.TTCommit1)() + + return t.Worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) +} + +func (t *trackedWorker) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) { + defer t.tracker.track(sector, sealtasks.TTCommit2)() + + return t.Worker.SealCommit2(ctx, sector, c1o) +} + +func (t *trackedWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { + defer t.tracker.track(sector, sealtasks.TTFinalize)() + + return t.Worker.FinalizeSector(ctx, sector, keepUnsealed) +} + +func (t *trackedWorker) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) { + defer t.tracker.track(sector, sealtasks.TTAddPiece)() + + return t.Worker.AddPiece(ctx, sector, pieceSizes, newPieceSize, pieceData) +} + +func (t *trackedWorker) Fetch(ctx context.Context, s abi.SectorID, ft stores.SectorFileType, ptype stores.PathType, am stores.AcquireMode) error { + defer t.tracker.track(s, sealtasks.TTFetch)() + + return t.Worker.Fetch(ctx, s, ft, ptype, am) +} + +func (t *trackedWorker) UnsealPiece(ctx context.Context, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) error { + defer t.tracker.track(id, sealtasks.TTUnseal)() + + return t.Worker.UnsealPiece(ctx, id, index, size, randomness, cid) +} + +func (t *trackedWorker) ReadPiece(ctx context.Context, writer io.Writer, id abi.SectorID, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + defer t.tracker.track(id, sealtasks.TTReadUnsealed)() + + return t.Worker.ReadPiece(ctx, writer, id, index, size) +} + +var _ Worker = &trackedWorker{} diff --git a/extern/sector-storage/zerocomm/zerocomm.go b/extern/sector-storage/zerocomm/zerocomm.go new file mode 100644 index 000000000..9b59723a0 --- /dev/null +++ b/extern/sector-storage/zerocomm/zerocomm.go @@ -0,0 +1,56 @@ +package zerocomm + +import ( + "math/bits" + + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-cid" +) + +const Levels = 37 +const Skip = 2 // can't generate for 32, 64b + +var PieceComms = [Levels - Skip][32]byte{ + {0x37, 0x31, 0xbb, 0x99, 0xac, 0x68, 0x9f, 0x66, 0xee, 0xf5, 0x97, 0x3e, 0x4a, 0x94, 0xda, 0x18, 0x8f, 0x4d, 0xdc, 0xae, 0x58, 0x7, 0x24, 0xfc, 0x6f, 0x3f, 0xd6, 0xd, 0xfd, 0x48, 0x83, 0x33}, + {0x64, 0x2a, 0x60, 0x7e, 0xf8, 0x86, 0xb0, 0x4, 0xbf, 0x2c, 0x19, 0x78, 0x46, 0x3a, 0xe1, 0xd4, 0x69, 0x3a, 0xc0, 0xf4, 0x10, 0xeb, 0x2d, 0x1b, 0x7a, 0x47, 0xfe, 0x20, 0x5e, 0x5e, 0x75, 0xf}, + {0x57, 0xa2, 0x38, 0x1a, 0x28, 0x65, 0x2b, 0xf4, 0x7f, 0x6b, 0xef, 0x7a, 0xca, 0x67, 0x9b, 0xe4, 0xae, 0xde, 0x58, 0x71, 0xab, 0x5c, 0xf3, 0xeb, 0x2c, 0x8, 0x11, 0x44, 0x88, 0xcb, 0x85, 0x26}, + {0x1f, 0x7a, 0xc9, 0x59, 0x55, 0x10, 0xe0, 0x9e, 0xa4, 0x1c, 0x46, 0xb, 0x17, 0x64, 0x30, 0xbb, 0x32, 0x2c, 0xd6, 0xfb, 0x41, 0x2e, 0xc5, 0x7c, 0xb1, 0x7d, 0x98, 0x9a, 0x43, 0x10, 0x37, 0x2f}, + {0xfc, 0x7e, 0x92, 0x82, 0x96, 0xe5, 0x16, 0xfa, 0xad, 0xe9, 0x86, 0xb2, 0x8f, 0x92, 0xd4, 0x4a, 0x4f, 0x24, 0xb9, 0x35, 0x48, 0x52, 0x23, 0x37, 0x6a, 0x79, 0x90, 0x27, 0xbc, 0x18, 0xf8, 0x33}, + {0x8, 0xc4, 0x7b, 0x38, 0xee, 0x13, 0xbc, 0x43, 0xf4, 0x1b, 0x91, 0x5c, 0xe, 0xed, 0x99, 0x11, 0xa2, 0x60, 0x86, 0xb3, 0xed, 0x62, 0x40, 0x1b, 0xf9, 0xd5, 0x8b, 0x8d, 0x19, 0xdf, 0xf6, 0x24}, + {0xb2, 0xe4, 0x7b, 0xfb, 0x11, 0xfa, 0xcd, 0x94, 0x1f, 0x62, 0xaf, 0x5c, 0x75, 0xf, 0x3e, 0xa5, 0xcc, 0x4d, 0xf5, 0x17, 0xd5, 0xc4, 0xf1, 0x6d, 0xb2, 0xb4, 0xd7, 0x7b, 0xae, 0xc1, 0xa3, 0x2f}, + {0xf9, 0x22, 0x61, 0x60, 0xc8, 0xf9, 0x27, 0xbf, 0xdc, 0xc4, 0x18, 0xcd, 0xf2, 0x3, 0x49, 0x31, 0x46, 0x0, 0x8e, 0xae, 0xfb, 0x7d, 0x2, 0x19, 0x4d, 0x5e, 0x54, 0x81, 0x89, 0x0, 0x51, 0x8}, + {0x2c, 0x1a, 0x96, 0x4b, 0xb9, 0xb, 0x59, 0xeb, 0xfe, 0xf, 0x6d, 0xa2, 0x9a, 0xd6, 0x5a, 0xe3, 0xe4, 0x17, 0x72, 0x4a, 0x8f, 0x7c, 0x11, 0x74, 0x5a, 0x40, 0xca, 0xc1, 0xe5, 0xe7, 0x40, 0x11}, + {0xfe, 0xe3, 0x78, 0xce, 0xf1, 0x64, 0x4, 0xb1, 0x99, 0xed, 0xe0, 0xb1, 0x3e, 0x11, 0xb6, 0x24, 0xff, 0x9d, 0x78, 0x4f, 0xbb, 0xed, 0x87, 0x8d, 0x83, 0x29, 0x7e, 0x79, 0x5e, 0x2, 0x4f, 0x2}, + {0x8e, 0x9e, 0x24, 0x3, 0xfa, 0x88, 0x4c, 0xf6, 0x23, 0x7f, 0x60, 0xdf, 0x25, 0xf8, 0x3e, 0xe4, 0xd, 0xca, 0x9e, 0xd8, 0x79, 0xeb, 0x6f, 0x63, 0x52, 0xd1, 0x50, 0x84, 0xf5, 0xad, 0xd, 0x3f}, + {0x75, 0x2d, 0x96, 0x93, 0xfa, 0x16, 0x75, 0x24, 0x39, 0x54, 0x76, 0xe3, 0x17, 0xa9, 0x85, 0x80, 0xf0, 0x9, 0x47, 0xaf, 0xb7, 0xa3, 0x5, 0x40, 0xd6, 0x25, 0xa9, 0x29, 0x1c, 0xc1, 0x2a, 0x7}, + {0x70, 0x22, 0xf6, 0xf, 0x7e, 0xf6, 0xad, 0xfa, 0x17, 0x11, 0x7a, 0x52, 0x61, 0x9e, 0x30, 0xce, 0xa8, 0x2c, 0x68, 0x7, 0x5a, 0xdf, 0x1c, 0x66, 0x77, 0x86, 0xec, 0x50, 0x6e, 0xef, 0x2d, 0x19}, + {0xd9, 0x98, 0x87, 0xb9, 0x73, 0x57, 0x3a, 0x96, 0xe1, 0x13, 0x93, 0x64, 0x52, 0x36, 0xc1, 0x7b, 0x1f, 0x4c, 0x70, 0x34, 0xd7, 0x23, 0xc7, 0xa9, 0x9f, 0x70, 0x9b, 0xb4, 0xda, 0x61, 0x16, 0x2b}, + {0xd0, 0xb5, 0x30, 0xdb, 0xb0, 0xb4, 0xf2, 0x5c, 0x5d, 0x2f, 0x2a, 0x28, 0xdf, 0xee, 0x80, 0x8b, 0x53, 0x41, 0x2a, 0x2, 0x93, 0x1f, 0x18, 0xc4, 0x99, 0xf5, 0xa2, 0x54, 0x8, 0x6b, 0x13, 0x26}, + {0x84, 0xc0, 0x42, 0x1b, 0xa0, 0x68, 0x5a, 0x1, 0xbf, 0x79, 0x5a, 0x23, 0x44, 0x6, 0x4f, 0xe4, 0x24, 0xbd, 0x52, 0xa9, 0xd2, 0x43, 0x77, 0xb3, 0x94, 0xff, 0x4c, 0x4b, 0x45, 0x68, 0xe8, 0x11}, + {0x65, 0xf2, 0x9e, 0x5d, 0x98, 0xd2, 0x46, 0xc3, 0x8b, 0x38, 0x8c, 0xfc, 0x6, 0xdb, 0x1f, 0x6b, 0x2, 0x13, 0x3, 0xc5, 0xa2, 0x89, 0x0, 0xb, 0xdc, 0xe8, 0x32, 0xa9, 0xc3, 0xec, 0x42, 0x1c}, + {0xa2, 0x24, 0x75, 0x8, 0x28, 0x58, 0x50, 0x96, 0x5b, 0x7e, 0x33, 0x4b, 0x31, 0x27, 0xb0, 0xc0, 0x42, 0xb1, 0xd0, 0x46, 0xdc, 0x54, 0x40, 0x21, 0x37, 0x62, 0x7c, 0xd8, 0x79, 0x9c, 0xe1, 0x3a}, + {0xda, 0xfd, 0xab, 0x6d, 0xa9, 0x36, 0x44, 0x53, 0xc2, 0x6d, 0x33, 0x72, 0x6b, 0x9f, 0xef, 0xe3, 0x43, 0xbe, 0x8f, 0x81, 0x64, 0x9e, 0xc0, 0x9, 0xaa, 0xd3, 0xfa, 0xff, 0x50, 0x61, 0x75, 0x8}, + {0xd9, 0x41, 0xd5, 0xe0, 0xd6, 0x31, 0x4a, 0x99, 0x5c, 0x33, 0xff, 0xbd, 0x4f, 0xbe, 0x69, 0x11, 0x8d, 0x73, 0xd4, 0xe5, 0xfd, 0x2c, 0xd3, 0x1f, 0xf, 0x7c, 0x86, 0xeb, 0xdd, 0x14, 0xe7, 0x6}, + {0x51, 0x4c, 0x43, 0x5c, 0x3d, 0x4, 0xd3, 0x49, 0xa5, 0x36, 0x5f, 0xbd, 0x59, 0xff, 0xc7, 0x13, 0x62, 0x91, 0x11, 0x78, 0x59, 0x91, 0xc1, 0xa3, 0xc5, 0x3a, 0xf2, 0x20, 0x79, 0x74, 0x1a, 0x2f}, + {0xad, 0x6, 0x85, 0x39, 0x69, 0xd3, 0x7d, 0x34, 0xff, 0x8, 0xe0, 0x9f, 0x56, 0x93, 0xa, 0x4a, 0xd1, 0x9a, 0x89, 0xde, 0xf6, 0xc, 0xbf, 0xee, 0x7e, 0x1d, 0x33, 0x81, 0xc1, 0xe7, 0x1c, 0x37}, + {0x39, 0x56, 0xe, 0x7b, 0x13, 0xa9, 0x3b, 0x7, 0xa2, 0x43, 0xfd, 0x27, 0x20, 0xff, 0xa7, 0xcb, 0x3e, 0x1d, 0x2e, 0x50, 0x5a, 0xb3, 0x62, 0x9e, 0x79, 0xf4, 0x63, 0x13, 0x51, 0x2c, 0xda, 0x6}, + {0xcc, 0xc3, 0xc0, 0x12, 0xf5, 0xb0, 0x5e, 0x81, 0x1a, 0x2b, 0xbf, 0xdd, 0xf, 0x68, 0x33, 0xb8, 0x42, 0x75, 0xb4, 0x7b, 0xf2, 0x29, 0xc0, 0x5, 0x2a, 0x82, 0x48, 0x4f, 0x3c, 0x1a, 0x5b, 0x3d}, + {0x7d, 0xf2, 0x9b, 0x69, 0x77, 0x31, 0x99, 0xe8, 0xf2, 0xb4, 0xb, 0x77, 0x91, 0x9d, 0x4, 0x85, 0x9, 0xee, 0xd7, 0x68, 0xe2, 0xc7, 0x29, 0x7b, 0x1f, 0x14, 0x37, 0x3, 0x4f, 0xc3, 0xc6, 0x2c}, + {0x66, 0xce, 0x5, 0xa3, 0x66, 0x75, 0x52, 0xcf, 0x45, 0xc0, 0x2b, 0xcc, 0x4e, 0x83, 0x92, 0x91, 0x9b, 0xde, 0xac, 0x35, 0xde, 0x2f, 0xf5, 0x62, 0x71, 0x84, 0x8e, 0x9f, 0x7b, 0x67, 0x51, 0x7}, + {0xd8, 0x61, 0x2, 0x18, 0x42, 0x5a, 0xb5, 0xe9, 0x5b, 0x1c, 0xa6, 0x23, 0x9d, 0x29, 0xa2, 0xe4, 0x20, 0xd7, 0x6, 0xa9, 0x6f, 0x37, 0x3e, 0x2f, 0x9c, 0x9a, 0x91, 0xd7, 0x59, 0xd1, 0x9b, 0x1}, + {0x6d, 0x36, 0x4b, 0x1e, 0xf8, 0x46, 0x44, 0x1a, 0x5a, 0x4a, 0x68, 0x86, 0x23, 0x14, 0xac, 0xc0, 0xa4, 0x6f, 0x1, 0x67, 0x17, 0xe5, 0x34, 0x43, 0xe8, 0x39, 0xee, 0xdf, 0x83, 0xc2, 0x85, 0x3c}, + {0x7, 0x7e, 0x5f, 0xde, 0x35, 0xc5, 0xa, 0x93, 0x3, 0xa5, 0x50, 0x9, 0xe3, 0x49, 0x8a, 0x4e, 0xbe, 0xdf, 0xf3, 0x9c, 0x42, 0xb7, 0x10, 0xb7, 0x30, 0xd8, 0xec, 0x7a, 0xc7, 0xaf, 0xa6, 0x3e}, + {0xe6, 0x40, 0x5, 0xa6, 0xbf, 0xe3, 0x77, 0x79, 0x53, 0xb8, 0xad, 0x6e, 0xf9, 0x3f, 0xf, 0xca, 0x10, 0x49, 0xb2, 0x4, 0x16, 0x54, 0xf2, 0xa4, 0x11, 0xf7, 0x70, 0x27, 0x99, 0xce, 0xce, 0x2}, + {0x25, 0x9d, 0x3d, 0x6b, 0x1f, 0x4d, 0x87, 0x6d, 0x11, 0x85, 0xe1, 0x12, 0x3a, 0xf6, 0xf5, 0x50, 0x1a, 0xf0, 0xf6, 0x7c, 0xf1, 0x5b, 0x52, 0x16, 0x25, 0x5b, 0x7b, 0x17, 0x8d, 0x12, 0x5, 0x1d}, + {0x3f, 0x9a, 0x4d, 0x41, 0x1d, 0xa4, 0xef, 0x1b, 0x36, 0xf3, 0x5f, 0xf0, 0xa1, 0x95, 0xae, 0x39, 0x2a, 0xb2, 0x3f, 0xee, 0x79, 0x67, 0xb7, 0xc4, 0x1b, 0x3, 0xd1, 0x61, 0x3f, 0xc2, 0x92, 0x39}, + {0xfe, 0x4e, 0xf3, 0x28, 0xc6, 0x1a, 0xa3, 0x9c, 0xfd, 0xb2, 0x48, 0x4e, 0xaa, 0x32, 0xa1, 0x51, 0xb1, 0xfe, 0x3d, 0xfd, 0x1f, 0x96, 0xdd, 0x8c, 0x97, 0x11, 0xfd, 0x86, 0xd6, 0xc5, 0x81, 0x13}, + {0xf5, 0x5d, 0x68, 0x90, 0xe, 0x2d, 0x83, 0x81, 0xec, 0xcb, 0x81, 0x64, 0xcb, 0x99, 0x76, 0xf2, 0x4b, 0x2d, 0xe0, 0xdd, 0x61, 0xa3, 0x1b, 0x97, 0xce, 0x6e, 0xb2, 0x38, 0x50, 0xd5, 0xe8, 0x19}, + {0xaa, 0xaa, 0x8c, 0x4c, 0xb4, 0xa, 0xac, 0xee, 0x1e, 0x2, 0xdc, 0x65, 0x42, 0x4b, 0x2a, 0x6c, 0x8e, 0x99, 0xf8, 0x3, 0xb7, 0x2f, 0x79, 0x29, 0xc4, 0x10, 0x1d, 0x7f, 0xae, 0x6b, 0xff, 0x32}, +} + +func ZeroPieceCommitment(sz abi.UnpaddedPieceSize) cid.Cid { + level := bits.TrailingZeros64(uint64(sz.Padded())) - Skip - 5 // 2^5 = 32 + commP, _ := commcid.PieceCommitmentV1ToCID(PieceComms[level][:]) + return commP +} diff --git a/extern/sector-storage/zerocomm/zerocomm_test.go b/extern/sector-storage/zerocomm/zerocomm_test.go new file mode 100644 index 000000000..f3206740b --- /dev/null +++ b/extern/sector-storage/zerocomm/zerocomm_test.go @@ -0,0 +1,115 @@ +package zerocomm_test + +import ( + "bytes" + "fmt" + "io" + "testing" + + commcid "github.com/filecoin-project/go-fil-commcid" + abi "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/zerocomm" +) + +func TestComms(t *testing.T) { + t.Skip("don't have enough ram") // no, but seriously, currently this needs like 3tb of /tmp + + var expPieceComms [zerocomm.Levels - zerocomm.Skip]cid.Cid + + { + l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127) + if err != nil { + t.Fatal(err) + } + expPieceComms[0] = l2 + } + + for i := 1; i < zerocomm.Levels-2; i++ { + var err error + sz := abi.UnpaddedPieceSize(127 << uint(i)) + fmt.Println(i, sz) + r := io.LimitReader(&NullReader{}, int64(sz)) + + expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz) + if err != nil { + t.Fatal(err) + } + } + + for i, comm := range expPieceComms { + c, err := commcid.CIDToPieceCommitmentV1(comm) + if err != nil { + t.Fatal(err) + } + if string(c) != string(zerocomm.PieceComms[i][:]) { + t.Errorf("zero commitment %d didn't match", i) + } + } + + for _, comm := range expPieceComms { // Could do codegen, but this is good enough + fmt.Printf("%#v,\n", comm) + } +} + +func TestCommsSmall(t *testing.T) { + var expPieceComms [8]cid.Cid + lvls := len(expPieceComms) + zerocomm.Skip + + { + l2, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 127)), 127) + if err != nil { + t.Fatal(err) + } + expPieceComms[0] = l2 + } + + for i := 1; i < lvls-2; i++ { + var err error + sz := abi.UnpaddedPieceSize(127 << uint(i)) + fmt.Println(i, sz) + r := io.LimitReader(&NullReader{}, int64(sz)) + + expPieceComms[i], err = ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, r, sz) + if err != nil { + t.Fatal(err) + } + } + + for i, comm := range expPieceComms { + c, err := commcid.CIDToPieceCommitmentV1(comm) + if err != nil { + t.Fatal(err) + } + if string(c) != string(zerocomm.PieceComms[i][:]) { + t.Errorf("zero commitment %d didn't match", i) + } + } + + for _, comm := range expPieceComms { // Could do codegen, but this is good enough + fmt.Printf("%#v,\n", comm) + } +} + +func TestForSise(t *testing.T) { + exp, err := ffiwrapper.GeneratePieceCIDFromFile(abi.RegisteredSealProof_StackedDrg2KiBV1, bytes.NewReader(make([]byte, 1016)), 1016) + if err != nil { + return + } + + actual := zerocomm.ZeroPieceCommitment(1016) + if !exp.Equals(actual) { + t.Errorf("zero commitment didn't match") + } +} + +type NullReader struct{} + +func (NullReader) Read(out []byte) (int, error) { + for i := range out { + out[i] = 0 + } + return len(out), nil +} diff --git a/extern/storage-fsm/.circleci/config.yml b/extern/storage-fsm/.circleci/config.yml new file mode 100644 index 000000000..a7cb9a24b --- /dev/null +++ b/extern/storage-fsm/.circleci/config.yml @@ -0,0 +1,79 @@ +version: 2.1 +orbs: + go: gotest/tools@0.0.9 +executors: + golang: + docker: + - image: circleci/golang:1.13 + resource_class: 2xlarge +commands: + prepare-git-checkout: + steps: + - checkout + - run: git submodule sync + - run: git submodule update --init --recursive + install-build-dependencies: + steps: + - run: sudo apt-get update + - run: sudo apt-get install -y jq ocl-icd-opencl-dev + - run: ./extern/filecoin-ffi/install-filcrypto + download-groth-params-and-verifying-keys: + steps: + - restore_cache: + name: Restore parameters cache + keys: + - 'v24-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ + - run: | + DIR=$(pwd) + cd $(mktemp -d) + go get github.com/filecoin-project/go-paramfetch/paramfetch + go build -o go-paramfetch github.com/filecoin-project/go-paramfetch/paramfetch + ./go-paramfetch 2048 "${DIR}/parameters.json" + - save_cache: + name: Save parameters cache + key: 'v24-2k-lotus-params' + paths: + - /var/tmp/filecoin-proof-parameters/ +jobs: + test: + executor: golang + environment: + RUST_LOG: info + steps: + - prepare-git-checkout + - install-build-dependencies + - download-groth-params-and-verifying-keys + - run: go test -v -timeout 10m ./... + mod-tidy-check: + executor: golang + steps: + - prepare-git-checkout + - go/mod-download + - go/mod-tidy-check + gofmt-check: + executor: golang + steps: + - prepare-git-checkout + - go/mod-download + - run: "! go fmt ./... 2>&1 | read" + lint-check: + executor: golang + steps: + - prepare-git-checkout + - install-build-dependencies + - go/mod-download + - go/install-golangci-lint: + gobin: $HOME/.local/bin + version: 1.23.8 + - run: + command: $HOME/.local/bin/golangci-lint run -v --concurrency 2 +workflows: + version: 2.1 + build_and_test: + jobs: + - mod-tidy-check + - lint-check + - gofmt-check + - test diff --git a/extern/storage-fsm/.gitignore b/extern/storage-fsm/.gitignore new file mode 100644 index 000000000..72fda5716 --- /dev/null +++ b/extern/storage-fsm/.gitignore @@ -0,0 +1,24 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# filecoin-ffi assets +*.a +*.h +*.pc + +# build artifacts +build/.filecoin-ffi-install +build/.update-submodules diff --git a/extern/storage-fsm/.gitmodules b/extern/storage-fsm/.gitmodules new file mode 100644 index 000000000..c50b68575 --- /dev/null +++ b/extern/storage-fsm/.gitmodules @@ -0,0 +1,4 @@ +[submodule "extern/filecoin-ffi"] + path = extern/filecoin-ffi + url = git@github.com:filecoin-project/filecoin-ffi + branch = master diff --git a/extern/storage-fsm/LICENSE-APACHE b/extern/storage-fsm/LICENSE-APACHE new file mode 100644 index 000000000..14478a3b6 --- /dev/null +++ b/extern/storage-fsm/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/extern/storage-fsm/LICENSE-MIT b/extern/storage-fsm/LICENSE-MIT new file mode 100644 index 000000000..72dc60d84 --- /dev/null +++ b/extern/storage-fsm/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/extern/storage-fsm/Makefile b/extern/storage-fsm/Makefile new file mode 100644 index 000000000..d22fa2636 --- /dev/null +++ b/extern/storage-fsm/Makefile @@ -0,0 +1,59 @@ +SHELL=/usr/bin/env bash + +all: build +.PHONY: all + +# git submodules that need to be loaded +SUBMODULES:= + +# things to clean up, e.g. libfilecoin.a +CLEAN:= + +FFI_PATH:=extern/filecoin-ffi/ +FFI_DEPS:=libfilcrypto.a filcrypto.pc filcrypto.h +FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS)) + +$(FFI_DEPS): build/.filecoin-ffi-install ; + +# dummy file that marks the last time the filecoin-ffi project was built +build/.filecoin-ffi-install: $(FFI_PATH) + $(MAKE) -C $(FFI_PATH) $(FFI_DEPS:$(FFI_PATH)%=%) + @touch $@ + +SUBMODULES+=$(FFI_PATH) +BUILD_DEPS+=build/.filecoin-ffi-install +CLEAN+=build/.filecoin-ffi-install + +$(SUBMODULES): build/.update-submodules ; + +# dummy file that marks the last time submodules were updated +build/.update-submodules: + git submodule update --init --recursive + touch $@ + +CLEAN+=build/.update-submodules + +# build and install any upstream dependencies, e.g. filecoin-ffi +deps: $(BUILD_DEPS) +.PHONY: deps + +test: $(BUILD_DEPS) + RUST_LOG=info go test -race -count 1 -v -timeout 120m ./... +.PHONY: test + +lint: $(BUILD_DEPS) + golangci-lint run -v --concurrency 2 --new-from-rev origin/master +.PHONY: lint + +build: $(BUILD_DEPS) + go build -v $(GOFLAGS) ./... +.PHONY: build + +clean: + rm -rf $(CLEAN) + -$(MAKE) -C $(FFI_PATH) clean +.PHONY: clean + +type-gen: + go run ./gen/main.go +.PHONY: type-gen diff --git a/extern/storage-fsm/README.md b/extern/storage-fsm/README.md new file mode 100644 index 000000000..346bd4614 --- /dev/null +++ b/extern/storage-fsm/README.md @@ -0,0 +1,18 @@ +# storage-fsm + +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![CircleCI](https://circleci.com/gh/filecoin-project/storage-fsm.svg?style=svg)](https://circleci.com/gh/filecoin-project/storage-fsm) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) + +> A finite state machine used for sector storage + +## Disclaimer + +Please report your issues with regards to storage-fsm at the [lotus issue tracker](https://github.com/filecoin-project/lotus/issues) + +## License + +The Filecoin Project is dual-licensed under Apache 2.0 and MIT terms: + +- Apache License, Version 2.0, ([LICENSE-APACHE](https://github.com/filecoin-project/storage-fsm/blob/master/LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](https://github.com/filecoin-project/storage-fsm/blob/master/LICENSE-MIT) or http://opensource.org/licenses/MIT) diff --git a/extern/storage-fsm/build/.keep b/extern/storage-fsm/build/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/extern/storage-fsm/cbor_gen.go b/extern/storage-fsm/cbor_gen.go new file mode 100644 index 000000000..9410e6f28 --- /dev/null +++ b/extern/storage-fsm/cbor_gen.go @@ -0,0 +1,1563 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package sealing + +import ( + "fmt" + "io" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf + +func (t *Piece) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + // t.Piece (abi.PieceInfo) (struct) + if len("Piece") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Piece\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Piece")))); err != nil { + return err + } + if _, err := w.Write([]byte("Piece")); err != nil { + return err + } + + if err := t.Piece.MarshalCBOR(w); err != nil { + return err + } + + // t.DealInfo (sealing.DealInfo) (struct) + if len("DealInfo") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealInfo\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealInfo")))); err != nil { + return err + } + if _, err := w.Write([]byte("DealInfo")); err != nil { + return err + } + + if err := t.DealInfo.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *Piece) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Piece: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Piece (abi.PieceInfo) (struct) + case "Piece": + + { + + if err := t.Piece.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Piece: %w", err) + } + + } + // t.DealInfo (sealing.DealInfo) (struct) + case "DealInfo": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.DealInfo = new(DealInfo) + if err := t.DealInfo.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err) + } + } + + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *DealInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{163}); err != nil { + return err + } + + // t.DealID (abi.DealID) (uint64) + if len("DealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealID\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealID")))); err != nil { + return err + } + if _, err := w.Write([]byte("DealID")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { + return err + } + + // t.DealSchedule (sealing.DealSchedule) (struct) + if len("DealSchedule") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealSchedule\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealSchedule")))); err != nil { + return err + } + if _, err := w.Write([]byte("DealSchedule")); err != nil { + return err + } + + if err := t.DealSchedule.MarshalCBOR(w); err != nil { + return err + } + + // t.KeepUnsealed (bool) (bool) + if len("KeepUnsealed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("KeepUnsealed")))); err != nil { + return err + } + if _, err := w.Write([]byte("KeepUnsealed")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.KeepUnsealed); err != nil { + return err + } + return nil +} + +func (t *DealInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealInfo: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DealID (abi.DealID) (uint64) + case "DealID": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = abi.DealID(extra) + + } + // t.DealSchedule (sealing.DealSchedule) (struct) + case "DealSchedule": + + { + + if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) + } + + } + // t.KeepUnsealed (bool) (bool) + case "KeepUnsealed": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.KeepUnsealed = false + case 21: + t.KeepUnsealed = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *DealSchedule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + // t.StartEpoch (abi.ChainEpoch) (int64) + if len("StartEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"StartEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("StartEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("StartEpoch")); err != nil { + return err + } + + if t.StartEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.StartEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.StartEpoch)-1)); err != nil { + return err + } + } + + // t.EndEpoch (abi.ChainEpoch) (int64) + if len("EndEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"EndEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("EndEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("EndEpoch")); err != nil { + return err + } + + if t.EndEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.EndEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.EndEpoch)-1)); err != nil { + return err + } + } + return nil +} + +func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StartEpoch (abi.ChainEpoch) (int64) + case "StartEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StartEpoch = abi.ChainEpoch(extraI) + } + // t.EndEpoch (abi.ChainEpoch) (int64) + case "EndEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.EndEpoch = abi.ChainEpoch(extraI) + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *SectorInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{182}); err != nil { + return err + } + + // t.State (sealing.SectorState) (string) + if len("State") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"State\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("State")))); err != nil { + return err + } + if _, err := w.Write([]byte("State")); err != nil { + return err + } + + if len(t.State) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.State was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.State)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.State)); err != nil { + return err + } + + // t.SectorNumber (abi.SectorNumber) (uint64) + if len("SectorNumber") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorNumber\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorNumber")))); err != nil { + return err + } + if _, err := w.Write([]byte("SectorNumber")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorNumber))); err != nil { + return err + } + + // t.SectorType (abi.RegisteredSealProof) (int64) + if len("SectorType") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SectorType\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorType")))); err != nil { + return err + } + if _, err := w.Write([]byte("SectorType")); err != nil { + return err + } + + if t.SectorType >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorType))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.SectorType)-1)); err != nil { + return err + } + } + + // t.Pieces ([]sealing.Piece) (slice) + if len("Pieces") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Pieces\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Pieces")))); err != nil { + return err + } + if _, err := w.Write([]byte("Pieces")); err != nil { + return err + } + + if len(t.Pieces) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Pieces was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { + return err + } + for _, v := range t.Pieces { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.TicketValue (abi.SealRandomness) (slice) + if len("TicketValue") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketValue\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketValue")))); err != nil { + return err + } + if _, err := w.Write([]byte("TicketValue")); err != nil { + return err + } + + if len(t.TicketValue) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.TicketValue was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.TicketValue)))); err != nil { + return err + } + if _, err := w.Write(t.TicketValue); err != nil { + return err + } + + // t.TicketEpoch (abi.ChainEpoch) (int64) + if len("TicketEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"TicketEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("TicketEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("TicketEpoch")); err != nil { + return err + } + + if t.TicketEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TicketEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.TicketEpoch)-1)); err != nil { + return err + } + } + + // t.PreCommit1Out (storage.PreCommit1Out) (slice) + if len("PreCommit1Out") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommit1Out\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommit1Out")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommit1Out")); err != nil { + return err + } + + if len(t.PreCommit1Out) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.PreCommit1Out was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PreCommit1Out)))); err != nil { + return err + } + if _, err := w.Write(t.PreCommit1Out); err != nil { + return err + } + + // t.CommD (cid.Cid) (struct) + if len("CommD") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommD\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommD")))); err != nil { + return err + } + if _, err := w.Write([]byte("CommD")); err != nil { + return err + } + + if t.CommD == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommD); err != nil { + return xerrors.Errorf("failed to write cid field t.CommD: %w", err) + } + } + + // t.CommR (cid.Cid) (struct) + if len("CommR") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommR\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommR")))); err != nil { + return err + } + if _, err := w.Write([]byte("CommR")); err != nil { + return err + } + + if t.CommR == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommR); err != nil { + return xerrors.Errorf("failed to write cid field t.CommR: %w", err) + } + } + + // t.Proof ([]uint8) (slice) + if len("Proof") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Proof\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Proof")))); err != nil { + return err + } + if _, err := w.Write([]byte("Proof")); err != nil { + return err + } + + if len(t.Proof) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Proof was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Proof)))); err != nil { + return err + } + if _, err := w.Write(t.Proof); err != nil { + return err + } + + // t.PreCommitInfo (miner.SectorPreCommitInfo) (struct) + if len("PreCommitInfo") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitInfo\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitInfo")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommitInfo")); err != nil { + return err + } + + if err := t.PreCommitInfo.MarshalCBOR(w); err != nil { + return err + } + + // t.PreCommitDeposit (big.Int) (struct) + if len("PreCommitDeposit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitDeposit\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitDeposit")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommitDeposit")); err != nil { + return err + } + + if err := t.PreCommitDeposit.MarshalCBOR(w); err != nil { + return err + } + + // t.PreCommitMessage (cid.Cid) (struct) + if len("PreCommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitMessage\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitMessage")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommitMessage")); err != nil { + return err + } + + if t.PreCommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.PreCommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommitMessage: %w", err) + } + } + + // t.PreCommitTipSet (sealing.TipSetToken) (slice) + if len("PreCommitTipSet") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommitTipSet\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommitTipSet")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommitTipSet")); err != nil { + return err + } + + if len(t.PreCommitTipSet) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.PreCommitTipSet was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PreCommitTipSet)))); err != nil { + return err + } + if _, err := w.Write(t.PreCommitTipSet); err != nil { + return err + } + + // t.PreCommit2Fails (uint64) (uint64) + if len("PreCommit2Fails") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PreCommit2Fails\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PreCommit2Fails")))); err != nil { + return err + } + if _, err := w.Write([]byte("PreCommit2Fails")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PreCommit2Fails))); err != nil { + return err + } + + // t.SeedValue (abi.InteractiveSealRandomness) (slice) + if len("SeedValue") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SeedValue\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SeedValue")))); err != nil { + return err + } + if _, err := w.Write([]byte("SeedValue")); err != nil { + return err + } + + if len(t.SeedValue) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.SeedValue was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.SeedValue)))); err != nil { + return err + } + if _, err := w.Write(t.SeedValue); err != nil { + return err + } + + // t.SeedEpoch (abi.ChainEpoch) (int64) + if len("SeedEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"SeedEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SeedEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("SeedEpoch")); err != nil { + return err + } + + if t.SeedEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SeedEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.SeedEpoch)-1)); err != nil { + return err + } + } + + // t.CommitMessage (cid.Cid) (struct) + if len("CommitMessage") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CommitMessage\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommitMessage")))); err != nil { + return err + } + if _, err := w.Write([]byte("CommitMessage")); err != nil { + return err + } + + if t.CommitMessage == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.CommitMessage); err != nil { + return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) + } + } + + // t.InvalidProofs (uint64) (uint64) + if len("InvalidProofs") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"InvalidProofs\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("InvalidProofs")))); err != nil { + return err + } + if _, err := w.Write([]byte("InvalidProofs")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.InvalidProofs))); err != nil { + return err + } + + // t.FaultReportMsg (cid.Cid) (struct) + if len("FaultReportMsg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"FaultReportMsg\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("FaultReportMsg")))); err != nil { + return err + } + if _, err := w.Write([]byte("FaultReportMsg")); err != nil { + return err + } + + if t.FaultReportMsg == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.FaultReportMsg); err != nil { + return xerrors.Errorf("failed to write cid field t.FaultReportMsg: %w", err) + } + } + + // t.LastErr (string) (string) + if len("LastErr") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"LastErr\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("LastErr")))); err != nil { + return err + } + if _, err := w.Write([]byte("LastErr")); err != nil { + return err + } + + if len(t.LastErr) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.LastErr was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.LastErr)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.LastErr)); err != nil { + return err + } + + // t.Log ([]sealing.Log) (slice) + if len("Log") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Log\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Log")))); err != nil { + return err + } + if _, err := w.Write([]byte("Log")); err != nil { + return err + } + + if len(t.Log) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Log was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Log)))); err != nil { + return err + } + for _, v := range t.Log { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + return nil +} + +func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorInfo: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.State (sealing.SectorState) (string) + case "State": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.State = SectorState(sval) + } + // t.SectorNumber (abi.SectorNumber) (uint64) + case "SectorNumber": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = abi.SectorNumber(extra) + + } + // t.SectorType (abi.RegisteredSealProof) (int64) + case "SectorType": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.SectorType = abi.RegisteredSealProof(extraI) + } + // t.Pieces ([]sealing.Piece) (slice) + case "Pieces": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Pieces: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Pieces = make([]Piece, extra) + } + + for i := 0; i < int(extra); i++ { + + var v Piece + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Pieces[i] = v + } + + // t.TicketValue (abi.SealRandomness) (slice) + case "TicketValue": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.TicketValue: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketValue = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketValue); err != nil { + return err + } + // t.TicketEpoch (abi.ChainEpoch) (int64) + case "TicketEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.TicketEpoch = abi.ChainEpoch(extraI) + } + // t.PreCommit1Out (storage.PreCommit1Out) (slice) + case "PreCommit1Out": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PreCommit1Out: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PreCommit1Out = make([]byte, extra) + if _, err := io.ReadFull(br, t.PreCommit1Out); err != nil { + return err + } + // t.CommD (cid.Cid) (struct) + case "CommD": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommD: %w", err) + } + + t.CommD = &c + } + + } + // t.CommR (cid.Cid) (struct) + case "CommR": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommR: %w", err) + } + + t.CommR = &c + } + + } + // t.Proof ([]uint8) (slice) + case "Proof": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.PreCommitInfo (miner.SectorPreCommitInfo) (struct) + case "PreCommitInfo": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.PreCommitInfo = new(miner.SectorPreCommitInfo) + if err := t.PreCommitInfo.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.PreCommitInfo pointer: %w", err) + } + } + + } + // t.PreCommitDeposit (big.Int) (struct) + case "PreCommitDeposit": + + { + + if err := t.PreCommitDeposit.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.PreCommitDeposit: %w", err) + } + + } + // t.PreCommitMessage (cid.Cid) (struct) + case "PreCommitMessage": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) + } + + t.PreCommitMessage = &c + } + + } + // t.PreCommitTipSet (sealing.TipSetToken) (slice) + case "PreCommitTipSet": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PreCommitTipSet: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PreCommitTipSet = make([]byte, extra) + if _, err := io.ReadFull(br, t.PreCommitTipSet); err != nil { + return err + } + // t.PreCommit2Fails (uint64) (uint64) + case "PreCommit2Fails": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PreCommit2Fails = uint64(extra) + + } + // t.SeedValue (abi.InteractiveSealRandomness) (slice) + case "SeedValue": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.SeedValue: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.SeedValue = make([]byte, extra) + if _, err := io.ReadFull(br, t.SeedValue); err != nil { + return err + } + // t.SeedEpoch (abi.ChainEpoch) (int64) + case "SeedEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.SeedEpoch = abi.ChainEpoch(extraI) + } + // t.CommitMessage (cid.Cid) (struct) + case "CommitMessage": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = &c + } + + } + // t.InvalidProofs (uint64) (uint64) + case "InvalidProofs": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.InvalidProofs = uint64(extra) + + } + // t.FaultReportMsg (cid.Cid) (struct) + case "FaultReportMsg": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) + } + + t.FaultReportMsg = &c + } + + } + // t.LastErr (string) (string) + case "LastErr": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.LastErr = string(sval) + } + // t.Log ([]sealing.Log) (slice) + case "Log": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.MaxLength+1 { // +1 placed here to recover broken state machines in calibration net; feel free to drop + return fmt.Errorf("t.Log: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Log = make([]Log, extra) + } + + for i := 0; i < int(extra); i++ { + + var v Log + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Log[i] = v + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *Log) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{164}); err != nil { + return err + } + + // t.Timestamp (uint64) (uint64) + if len("Timestamp") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Timestamp\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Timestamp")))); err != nil { + return err + } + if _, err := w.Write([]byte("Timestamp")); err != nil { + return err + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { + return err + } + + // t.Trace (string) (string) + if len("Trace") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Trace\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Trace")))); err != nil { + return err + } + if _, err := w.Write([]byte("Trace")); err != nil { + return err + } + + if len(t.Trace) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Trace was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Trace)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Trace)); err != nil { + return err + } + + // t.Message (string) (string) + if len("Message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Message\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Message")))); err != nil { + return err + } + if _, err := w.Write([]byte("Message")); err != nil { + return err + } + + if len(t.Message) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Message was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Message)); err != nil { + return err + } + + // t.Kind (string) (string) + if len("Kind") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Kind\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Kind")))); err != nil { + return err + } + if _, err := w.Write([]byte("Kind")); err != nil { + return err + } + + if len(t.Kind) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Kind was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Kind)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.Kind)); err != nil { + return err + } + return nil +} + +func (t *Log) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Log: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Timestamp (uint64) (uint64) + case "Timestamp": + + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Timestamp = uint64(extra) + + } + // t.Trace (string) (string) + case "Trace": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Trace = string(sval) + } + // t.Message (string) (string) + case "Message": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Message = string(sval) + } + // t.Kind (string) (string) + case "Kind": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.Kind = string(sval) + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} diff --git a/extern/storage-fsm/checks.go b/extern/storage-fsm/checks.go new file mode 100644 index 000000000..bfc292202 --- /dev/null +++ b/extern/storage-fsm/checks.go @@ -0,0 +1,165 @@ +package sealing + +import ( + "bytes" + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/sector-storage/zerocomm" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/crypto" +) + +// TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting +// We should implement some wait-for-api logic +type ErrApi struct{ error } + +type ErrInvalidDeals struct{ error } +type ErrInvalidPiece struct{ error } +type ErrExpiredDeals struct{ error } + +type ErrBadCommD struct{ error } +type ErrExpiredTicket struct{ error } +type ErrBadTicket struct{ error } +type ErrPrecommitOnChain struct{ error } + +type ErrBadSeed struct{ error } +type ErrInvalidProof struct{ error } +type ErrNoPrecommit struct{ error } + +func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { + tok, height, err := api.ChainHead(ctx) + if err != nil { + return &ErrApi{xerrors.Errorf("getting chain head: %w", err)} + } + + for i, p := range si.Pieces { + // if no deal is associated with the piece, ensure that we added it as + // filler (i.e. ensure that it has a zero PieceCID) + if p.DealInfo == nil { + exp := zerocomm.ZeroPieceCommitment(p.Piece.Size.Unpadded()) + if !p.Piece.PieceCID.Equals(exp) { + return &ErrInvalidPiece{xerrors.Errorf("sector %d piece %d had non-zero PieceCID %+v", si.SectorNumber, i, p.Piece.PieceCID)} + } + continue + } + + proposal, err := api.StateMarketStorageDeal(ctx, p.DealInfo.DealID, tok) + if err != nil { + return &ErrInvalidDeals{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)} + } + + if proposal.PieceCID != p.Piece.PieceCID { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, proposal.PieceCID)} + } + + if p.Piece.Size != proposal.PieceSize { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.Size, proposal.PieceSize)} + } + + if height >= proposal.StartEpoch { + return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, proposal.StartEpoch, height)} + } + } + + return nil +} + +// checkPrecommit checks that data commitment generated in the sealing process +// matches pieces, and that the seal ticket isn't expired +func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, tok TipSetToken, height abi.ChainEpoch, api SealingAPI) (err error) { + commD, err := api.StateComputeDataCommitment(ctx, maddr, si.SectorType, si.dealIDs(), tok) + if err != nil { + return &ErrApi{xerrors.Errorf("calling StateComputeDataCommitment: %w", err)} + } + + if !commD.Equals(*si.CommD) { + return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)} + } + + if height-(si.TicketEpoch+SealRandomnessLookback) > SealRandomnessLookbackLimit(si.SectorType) { + return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+SealRandomnessLookback, height)} + } + + pci, err := api.StateSectorPreCommitInfo(ctx, maddr, si.SectorNumber, tok) + if err != nil { + return &ErrApi{xerrors.Errorf("getting precommit info: %w", err)} + } + + if pci != nil { + if pci.Info.SealRandEpoch != si.TicketEpoch { + return &ErrBadTicket{xerrors.Errorf("bad ticket epoch: %d != %d", pci.Info.SealRandEpoch, si.TicketEpoch)} + } + return &ErrPrecommitOnChain{xerrors.Errorf("precommit already on chain")} + } + + return nil +} + +func (m *Sealing) checkCommit(ctx context.Context, si SectorInfo, proof []byte, tok TipSetToken) (err error) { + if si.SeedEpoch == 0 { + return &ErrBadSeed{xerrors.Errorf("seed epoch was not set")} + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx, m.maddr, si.SectorNumber, tok) + if err != nil { + return xerrors.Errorf("getting precommit info: %w", err) + } + + if pci == nil { + return &ErrNoPrecommit{xerrors.Errorf("precommit info not found on-chain")} + } + + if pci.PreCommitEpoch+miner.PreCommitChallengeDelay != si.SeedEpoch { + return &ErrBadSeed{xerrors.Errorf("seed epoch doesn't match on chain info: %d != %d", pci.PreCommitEpoch+miner.PreCommitChallengeDelay, si.SeedEpoch)} + } + + buf := new(bytes.Buffer) + if err := m.maddr.MarshalCBOR(buf); err != nil { + return err + } + + seed, err := m.api.ChainGetRandomness(ctx, tok, crypto.DomainSeparationTag_InteractiveSealChallengeSeed, si.SeedEpoch, buf.Bytes()) + if err != nil { + return &ErrApi{xerrors.Errorf("failed to get randomness for computing seal proof: %w", err)} + } + + if string(seed) != string(si.SeedValue) { + return &ErrBadSeed{xerrors.Errorf("seed has changed")} + } + + ss, err := m.api.StateMinerSectorSize(ctx, m.maddr, tok) + if err != nil { + return &ErrApi{err} + } + spt, err := ffiwrapper.SealProofTypeFromSectorSize(ss) + if err != nil { + return err + } + + if *si.CommR != pci.Info.SealedCID { + log.Warn("on-chain sealed CID doesn't match!") + } + + ok, err := m.verif.VerifySeal(abi.SealVerifyInfo{ + SectorID: m.minerSector(si.SectorNumber), + SealedCID: pci.Info.SealedCID, + SealProof: spt, + Proof: proof, + Randomness: si.TicketValue, + InteractiveRandomness: si.SeedValue, + UnsealedCID: *si.CommD, + }) + if err != nil { + return xerrors.Errorf("verify seal: %w", err) + } + if !ok { + return &ErrInvalidProof{xerrors.New("invalid proof (compute error?)")} + } + + return nil +} diff --git a/extern/storage-fsm/constants.go b/extern/storage-fsm/constants.go new file mode 100644 index 000000000..565a38c8e --- /dev/null +++ b/extern/storage-fsm/constants.go @@ -0,0 +1,17 @@ +package sealing + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +// Epochs +const SealRandomnessLookback = miner.ChainFinality + +// Epochs +func SealRandomnessLookbackLimit(spt abi.RegisteredSealProof) abi.ChainEpoch { + return miner.MaxSealDuration[spt] +} + +// Epochs +const InteractivePoRepConfidence = 6 diff --git a/extern/storage-fsm/events.go b/extern/storage-fsm/events.go new file mode 100644 index 000000000..ba6d2a860 --- /dev/null +++ b/extern/storage-fsm/events.go @@ -0,0 +1,15 @@ +package sealing + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +// `curH`-`ts.Height` = `confidence` +type HeightHandler func(ctx context.Context, tok TipSetToken, curH abi.ChainEpoch) error +type RevertHandler func(ctx context.Context, tok TipSetToken) error + +type Events interface { + ChainAt(hnd HeightHandler, rev RevertHandler, confidence int, h abi.ChainEpoch) error +} diff --git a/extern/storage-fsm/fsm.go b/extern/storage-fsm/fsm.go new file mode 100644 index 000000000..25b8f364a --- /dev/null +++ b/extern/storage-fsm/fsm.go @@ -0,0 +1,394 @@ +package sealing + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "reflect" + "time" + + "golang.org/x/xerrors" + + statemachine "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) { + next, err := m.plan(events, user.(*SectorInfo)) + if err != nil || next == nil { + return nil, uint64(len(events)), err + } + + return func(ctx statemachine.Context, si SectorInfo) error { + err := next(ctx, si) + if err != nil { + log.Errorf("unhandled sector error (%d): %+v", si.SectorNumber, err) + return nil + } + + return nil + }, uint64(len(events)), nil // TODO: This processed event count is not very correct +} + +var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *SectorInfo) error{ + // Sealing + + UndefinedSectorState: planOne( + on(SectorStart{}, Empty), + on(SectorStartCC{}, Packing), + ), + Empty: planOne(on(SectorAddPiece{}, WaitDeals)), + WaitDeals: planOne( + on(SectorAddPiece{}, WaitDeals), + on(SectorStartPacking{}, Packing), + ), + Packing: planOne(on(SectorPacked{}, PreCommit1)), + PreCommit1: planOne( + on(SectorPreCommit1{}, PreCommit2), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), + on(SectorPackingFailed{}, PackingFailed), + ), + PreCommit2: planOne( + on(SectorPreCommit2{}, PreCommitting), + on(SectorSealPreCommit2Failed{}, SealPreCommit2Failed), + on(SectorPackingFailed{}, PackingFailed), + ), + PreCommitting: planOne( + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), + on(SectorPreCommitted{}, PreCommitWait), + on(SectorChainPreCommitFailed{}, PreCommitFailed), + on(SectorPreCommitLanded{}, WaitSeed), + ), + PreCommitWait: planOne( + on(SectorChainPreCommitFailed{}, PreCommitFailed), + on(SectorPreCommitLanded{}, WaitSeed), + ), + WaitSeed: planOne( + on(SectorSeedReady{}, Committing), + on(SectorChainPreCommitFailed{}, PreCommitFailed), + ), + Committing: planCommitting, + CommitWait: planOne( + on(SectorProving{}, FinalizeSector), + on(SectorCommitFailed{}, CommitFailed), + ), + + FinalizeSector: planOne( + on(SectorFinalized{}, Proving), + on(SectorFinalizeFailed{}, FinalizeFailed), + ), + + // Sealing errors + + SealPreCommit1Failed: planOne( + on(SectorRetrySealPreCommit1{}, PreCommit1), + ), + SealPreCommit2Failed: planOne( + on(SectorRetrySealPreCommit1{}, PreCommit1), + on(SectorRetrySealPreCommit2{}, PreCommit2), + ), + PreCommitFailed: planOne( + on(SectorRetryPreCommit{}, PreCommitting), + on(SectorRetryWaitSeed{}, WaitSeed), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), + on(SectorPreCommitLanded{}, WaitSeed), + ), + ComputeProofFailed: planOne( + on(SectorRetryComputeProof{}, Committing), + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), + ), + CommitFailed: planOne( + on(SectorSealPreCommit1Failed{}, SealPreCommit1Failed), + on(SectorRetryWaitSeed{}, WaitSeed), + on(SectorRetryComputeProof{}, Committing), + on(SectorRetryInvalidProof{}, Committing), + on(SectorRetryPreCommitWait{}, PreCommitWait), + on(SectorChainPreCommitFailed{}, PreCommitFailed), + on(SectorRetryPreCommit{}, PreCommitting), + ), + FinalizeFailed: planOne( + on(SectorRetryFinalize{}, FinalizeSector), + ), + + // Post-seal + + Proving: planOne( + on(SectorFaultReported{}, FaultReported), + on(SectorFaulty{}, Faulty), + on(SectorRemove{}, Removing), + ), + Removing: planOne( + on(SectorRemoved{}, Removed), + on(SectorRemoveFailed{}, RemoveFailed), + ), + Faulty: planOne( + on(SectorFaultReported{}, FaultReported), + ), + + FaultedFinal: final, + Removed: final, +} + +func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, error) { + ///// + // First process all events + + for _, event := range events { + e, err := json.Marshal(event) + if err != nil { + log.Errorf("marshaling event for logging: %+v", err) + continue + } + + l := Log{ + Timestamp: uint64(time.Now().Unix()), + Message: string(e), + Kind: fmt.Sprintf("event;%T", event.User), + } + + if err, iserr := event.User.(xerrors.Formatter); iserr { + l.Trace = fmt.Sprintf("%+v", err) + } + + if len(state.Log) > 8000 { + log.Warnw("truncating sector log", "sector", state.SectorNumber) + state.Log[2000] = Log{ + Timestamp: uint64(time.Now().Unix()), + Message: "truncating log (above 8000 entries)", + Kind: fmt.Sprintf("truncate"), + } + + state.Log = append(state.Log[:2000], state.Log[:6000]...) + } + + state.Log = append(state.Log, l) + } + + p := fsmPlanners[state.State] + if p == nil { + return nil, xerrors.Errorf("planner for state %s not found", state.State) + } + + if err := p(events, state); err != nil { + return nil, xerrors.Errorf("running planner for state %s failed: %w", state.State, err) + } + + ///// + // Now decide what to do next + + /* + + * Empty <- incoming deals + | | + | v + *<- WaitDeals <- incoming deals + | | + | v + *<- Packing <- incoming committed capacity + | | + | v + *<- PreCommit1 <--> SealPreCommit1Failed + | | ^ ^^ + | | *----------++----\ + | v v || | + *<- PreCommit2 --------++--> SealPreCommit2Failed + | | || + | v /-------/| + * PreCommitting <-----+---> PreCommitFailed + | | | ^ + | v | | + *<- WaitSeed -----------+-----/ + | ||| ^ | + | ||| \--------*-----/ + | ||| | + | vvv v----+----> ComputeProofFailed + *<- Committing | + | | ^--> CommitFailed + | v ^ + *<- CommitWait ---/ + | | + | v + | FinalizeSector <--> FinalizeFailed + | | + | v + *<- Proving + | + v + FailedUnrecoverable + + UndefinedSectorState <- ¯\_(ツ)_/¯ + | ^ + *---------------------/ + + */ + + switch state.State { + // Happy path + case Empty: + fallthrough + case WaitDeals: + log.Infof("Waiting for deals %d", state.SectorNumber) + case Packing: + return m.handlePacking, nil + case PreCommit1: + return m.handlePreCommit1, nil + case PreCommit2: + return m.handlePreCommit2, nil + case PreCommitting: + return m.handlePreCommitting, nil + case PreCommitWait: + return m.handlePreCommitWait, nil + case WaitSeed: + return m.handleWaitSeed, nil + case Committing: + return m.handleCommitting, nil + case CommitWait: + return m.handleCommitWait, nil + case FinalizeSector: + return m.handleFinalizeSector, nil + + // Handled failure modes + case SealPreCommit1Failed: + return m.handleSealPrecommit1Failed, nil + case SealPreCommit2Failed: + return m.handleSealPrecommit2Failed, nil + case PreCommitFailed: + return m.handlePreCommitFailed, nil + case ComputeProofFailed: + return m.handleComputeProofFailed, nil + case CommitFailed: + return m.handleCommitFailed, nil + case FinalizeFailed: + return m.handleFinalizeFailed, nil + + // Post-seal + case Proving: + return m.handleProvingSector, nil + case Removing: + return m.handleRemoving, nil + case Removed: + return nil, nil + + // Faults + case Faulty: + return m.handleFaulty, nil + case FaultReported: + return m.handleFaultReported, nil + + // Fatal errors + case UndefinedSectorState: + log.Error("sector update with undefined state!") + case FailedUnrecoverable: + log.Errorf("sector %d failed unrecoverably", state.SectorNumber) + default: + log.Errorf("unexpected sector update state: %s", state.State) + } + + return nil, nil +} + +func planCommitting(events []statemachine.Event, state *SectorInfo) error { + for _, event := range events { + switch e := event.User.(type) { + case globalMutator: + if e.applyGlobal(state) { + return nil + } + case SectorCommitted: // the normal case + e.apply(state) + state.State = CommitWait + case SectorSeedReady: // seed changed :/ + if e.SeedEpoch == state.SeedEpoch && bytes.Equal(e.SeedValue, state.SeedValue) { + log.Warnf("planCommitting: got SectorSeedReady, but the seed didn't change") + continue // or it didn't! + } + log.Warnf("planCommitting: commit Seed changed") + e.apply(state) + state.State = Committing + return nil + case SectorComputeProofFailed: + state.State = ComputeProofFailed + case SectorSealPreCommit1Failed: + state.State = SealPreCommit1Failed + case SectorCommitFailed: + state.State = CommitFailed + default: + return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events) + } + } + return nil +} + +func (m *Sealing) restartSectors(ctx context.Context) error { + trackedSectors, err := m.ListSectors() + if err != nil { + log.Errorf("loading sector list: %+v", err) + } + + for _, sector := range trackedSectors { + if err := m.sectors.Send(uint64(sector.SectorNumber), SectorRestart{}); err != nil { + log.Errorf("restarting sector %d: %+v", sector.SectorNumber, err) + } + } + + // TODO: Grab on-chain sector set and diff with trackedSectors + + return nil +} + +func (m *Sealing) ForceSectorState(ctx context.Context, id abi.SectorNumber, state SectorState) error { + return m.sectors.Send(id, SectorForceState{state}) +} + +func final(events []statemachine.Event, state *SectorInfo) error { + return xerrors.Errorf("didn't expect any events in state %s, got %+v", state.State, events) +} + +func on(mut mutator, next SectorState) func() (mutator, SectorState) { + return func() (mutator, SectorState) { + return mut, next + } +} + +func planOne(ts ...func() (mut mutator, next SectorState)) func(events []statemachine.Event, state *SectorInfo) error { + return func(events []statemachine.Event, state *SectorInfo) error { + if len(events) != 1 { + for _, event := range events { + if gm, ok := event.User.(globalMutator); ok { + gm.applyGlobal(state) + return nil + } + } + return xerrors.Errorf("planner for state %s only has a plan for a single event only, got %+v", state.State, events) + } + + if gm, ok := events[0].User.(globalMutator); ok { + gm.applyGlobal(state) + return nil + } + + for _, t := range ts { + mut, next := t() + + if reflect.TypeOf(events[0].User) != reflect.TypeOf(mut) { + continue + } + + if err, iserr := events[0].User.(error); iserr { + log.Warnf("sector %d got error event %T: %+v", state.SectorNumber, events[0].User, err) + } + + events[0].User.(mutator).apply(state) + state.State = next + return nil + } + + _, ok := events[0].User.(Ignorable) + if ok { + return nil + } + + return xerrors.Errorf("planner for state %s received unexpected event %T (%+v)", state.State, events[0].User, events[0]) + } +} diff --git a/extern/storage-fsm/fsm_events.go b/extern/storage-fsm/fsm_events.go new file mode 100644 index 000000000..c4278991e --- /dev/null +++ b/extern/storage-fsm/fsm_events.go @@ -0,0 +1,282 @@ +package sealing + +import ( + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-storage/storage" +) + +type mutator interface { + apply(state *SectorInfo) +} + +// globalMutator is an event which can apply in every state +type globalMutator interface { + // applyGlobal applies the event to the state. If if returns true, + // event processing should be interrupted + applyGlobal(state *SectorInfo) bool +} + +type Ignorable interface { + Ignore() +} + +// Global events + +type SectorRestart struct{} + +func (evt SectorRestart) applyGlobal(*SectorInfo) bool { return false } + +type SectorFatalError struct{ error } + +func (evt SectorFatalError) FormatError(xerrors.Printer) (next error) { return evt.error } + +func (evt SectorFatalError) applyGlobal(state *SectorInfo) bool { + log.Errorf("Fatal error on sector %d: %+v", state.SectorNumber, evt.error) + // TODO: Do we want to mark the state as unrecoverable? + // I feel like this should be a softer error, where the user would + // be able to send a retry event of some kind + return true +} + +type SectorForceState struct { + State SectorState +} + +func (evt SectorForceState) applyGlobal(state *SectorInfo) bool { + state.State = evt.State + return true +} + +// Normal path + +type SectorStart struct { + ID abi.SectorNumber + SectorType abi.RegisteredSealProof +} + +func (evt SectorStart) apply(state *SectorInfo) { + state.SectorNumber = evt.ID + state.SectorType = evt.SectorType +} + +type SectorStartCC struct { + ID abi.SectorNumber + SectorType abi.RegisteredSealProof + Pieces []Piece +} + +func (evt SectorStartCC) apply(state *SectorInfo) { + state.SectorNumber = evt.ID + state.Pieces = evt.Pieces + state.SectorType = evt.SectorType +} + +type SectorAddPiece struct { + NewPiece Piece +} + +func (evt SectorAddPiece) apply(state *SectorInfo) { + state.Pieces = append(state.Pieces, evt.NewPiece) +} + +type SectorStartPacking struct{} + +func (evt SectorStartPacking) apply(*SectorInfo) {} + +func (evt SectorStartPacking) Ignore() {} + +type SectorPacked struct{ FillerPieces []abi.PieceInfo } + +func (evt SectorPacked) apply(state *SectorInfo) { + for idx := range evt.FillerPieces { + state.Pieces = append(state.Pieces, Piece{ + Piece: evt.FillerPieces[idx], + DealInfo: nil, // filler pieces don't have deals associated with them + }) + } +} + +type SectorPackingFailed struct{ error } + +func (evt SectorPackingFailed) apply(*SectorInfo) {} + +type SectorPreCommit1 struct { + PreCommit1Out storage.PreCommit1Out + TicketValue abi.SealRandomness + TicketEpoch abi.ChainEpoch +} + +func (evt SectorPreCommit1) apply(state *SectorInfo) { + state.PreCommit1Out = evt.PreCommit1Out + state.TicketEpoch = evt.TicketEpoch + state.TicketValue = evt.TicketValue + state.PreCommit2Fails = 0 +} + +type SectorPreCommit2 struct { + Sealed cid.Cid + Unsealed cid.Cid +} + +func (evt SectorPreCommit2) apply(state *SectorInfo) { + commd := evt.Unsealed + state.CommD = &commd + commr := evt.Sealed + state.CommR = &commr +} + +type SectorPreCommitLanded struct { + TipSet TipSetToken +} + +func (evt SectorPreCommitLanded) apply(si *SectorInfo) { + si.PreCommitTipSet = evt.TipSet +} + +type SectorSealPreCommit1Failed struct{ error } + +func (evt SectorSealPreCommit1Failed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorSealPreCommit1Failed) apply(si *SectorInfo) { + si.InvalidProofs = 0 // reset counter + si.PreCommit2Fails = 0 +} + +type SectorSealPreCommit2Failed struct{ error } + +func (evt SectorSealPreCommit2Failed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorSealPreCommit2Failed) apply(si *SectorInfo) { + si.InvalidProofs = 0 // reset counter + si.PreCommit2Fails++ +} + +type SectorChainPreCommitFailed struct{ error } + +func (evt SectorChainPreCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorChainPreCommitFailed) apply(*SectorInfo) {} + +type SectorPreCommitted struct { + Message cid.Cid + PreCommitDeposit big.Int + PreCommitInfo miner.SectorPreCommitInfo +} + +func (evt SectorPreCommitted) apply(state *SectorInfo) { + state.PreCommitMessage = &evt.Message + state.PreCommitDeposit = evt.PreCommitDeposit + state.PreCommitInfo = &evt.PreCommitInfo +} + +type SectorSeedReady struct { + SeedValue abi.InteractiveSealRandomness + SeedEpoch abi.ChainEpoch +} + +func (evt SectorSeedReady) apply(state *SectorInfo) { + state.SeedEpoch = evt.SeedEpoch + state.SeedValue = evt.SeedValue +} + +type SectorComputeProofFailed struct{ error } + +func (evt SectorComputeProofFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorComputeProofFailed) apply(*SectorInfo) {} + +type SectorCommitFailed struct{ error } + +func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorCommitFailed) apply(*SectorInfo) {} + +type SectorCommitted struct { + Message cid.Cid + Proof []byte +} + +func (evt SectorCommitted) apply(state *SectorInfo) { + state.Proof = evt.Proof + state.CommitMessage = &evt.Message +} + +type SectorProving struct{} + +func (evt SectorProving) apply(*SectorInfo) {} + +type SectorFinalized struct{} + +func (evt SectorFinalized) apply(*SectorInfo) {} + +type SectorRetryFinalize struct{} + +func (evt SectorRetryFinalize) apply(*SectorInfo) {} + +type SectorFinalizeFailed struct{ error } + +func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorFinalizeFailed) apply(*SectorInfo) {} + +// Failed state recovery + +type SectorRetrySealPreCommit1 struct{} + +func (evt SectorRetrySealPreCommit1) apply(state *SectorInfo) {} + +type SectorRetrySealPreCommit2 struct{} + +func (evt SectorRetrySealPreCommit2) apply(state *SectorInfo) {} + +type SectorRetryPreCommit struct{} + +func (evt SectorRetryPreCommit) apply(state *SectorInfo) {} + +type SectorRetryWaitSeed struct{} + +func (evt SectorRetryWaitSeed) apply(state *SectorInfo) {} + +type SectorRetryPreCommitWait struct{} + +func (evt SectorRetryPreCommitWait) apply(state *SectorInfo) {} + +type SectorRetryComputeProof struct{} + +func (evt SectorRetryComputeProof) apply(state *SectorInfo) { + state.InvalidProofs++ +} + +type SectorRetryInvalidProof struct{} + +func (evt SectorRetryInvalidProof) apply(state *SectorInfo) { + state.InvalidProofs++ +} + +// Faults + +type SectorFaulty struct{} + +func (evt SectorFaulty) apply(state *SectorInfo) {} + +type SectorFaultReported struct{ reportMsg cid.Cid } + +func (evt SectorFaultReported) apply(state *SectorInfo) { + state.FaultReportMsg = &evt.reportMsg +} + +type SectorFaultedFinal struct{} + +// External events + +type SectorRemove struct{} + +func (evt SectorRemove) apply(state *SectorInfo) {} + +type SectorRemoved struct{} + +func (evt SectorRemoved) apply(state *SectorInfo) {} + +type SectorRemoveFailed struct{ error } + +func (evt SectorRemoveFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorRemoveFailed) apply(*SectorInfo) {} diff --git a/extern/storage-fsm/fsm_test.go b/extern/storage-fsm/fsm_test.go new file mode 100644 index 000000000..b1e53133c --- /dev/null +++ b/extern/storage-fsm/fsm_test.go @@ -0,0 +1,115 @@ +package sealing + +import ( + "testing" + + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-statemachine" +) + +func init() { + _ = logging.SetLogLevel("*", "INFO") +} + +func (t *test) planSingle(evt interface{}) { + _, err := t.s.plan([]statemachine.Event{{User: evt}}, t.state) + require.NoError(t.t, err) +} + +type test struct { + s *Sealing + t *testing.T + state *SectorInfo +} + +func TestHappyPath(t *testing.T) { + m := test{ + s: &Sealing{}, + t: t, + state: &SectorInfo{State: Packing}, + } + + m.planSingle(SectorPacked{}) + require.Equal(m.t, m.state.State, PreCommit1) + + m.planSingle(SectorPreCommit1{}) + require.Equal(m.t, m.state.State, PreCommit2) + + m.planSingle(SectorPreCommit2{}) + require.Equal(m.t, m.state.State, PreCommitting) + + m.planSingle(SectorPreCommitted{}) + require.Equal(m.t, m.state.State, PreCommitWait) + + m.planSingle(SectorPreCommitLanded{}) + require.Equal(m.t, m.state.State, WaitSeed) + + m.planSingle(SectorSeedReady{}) + require.Equal(m.t, m.state.State, Committing) + + m.planSingle(SectorCommitted{}) + require.Equal(m.t, m.state.State, CommitWait) + + m.planSingle(SectorProving{}) + require.Equal(m.t, m.state.State, FinalizeSector) + + m.planSingle(SectorFinalized{}) + require.Equal(m.t, m.state.State, Proving) +} + +func TestSeedRevert(t *testing.T) { + m := test{ + s: &Sealing{}, + t: t, + state: &SectorInfo{State: Packing}, + } + + m.planSingle(SectorPacked{}) + require.Equal(m.t, m.state.State, PreCommit1) + + m.planSingle(SectorPreCommit1{}) + require.Equal(m.t, m.state.State, PreCommit2) + + m.planSingle(SectorPreCommit2{}) + require.Equal(m.t, m.state.State, PreCommitting) + + m.planSingle(SectorPreCommitted{}) + require.Equal(m.t, m.state.State, PreCommitWait) + + m.planSingle(SectorPreCommitLanded{}) + require.Equal(m.t, m.state.State, WaitSeed) + + m.planSingle(SectorSeedReady{}) + require.Equal(m.t, m.state.State, Committing) + + _, err := m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state) + require.NoError(t, err) + require.Equal(m.t, m.state.State, Committing) + + // not changing the seed this time + _, err = m.s.plan([]statemachine.Event{{User: SectorSeedReady{SeedValue: nil, SeedEpoch: 5}}, {User: SectorCommitted{}}}, m.state) + require.NoError(t, err) + require.Equal(m.t, m.state.State, CommitWait) + + m.planSingle(SectorProving{}) + require.Equal(m.t, m.state.State, FinalizeSector) + + m.planSingle(SectorFinalized{}) + require.Equal(m.t, m.state.State, Proving) +} + +func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) { + m := test{ + s: &Sealing{}, + t: t, + state: &SectorInfo{State: Committing}, + } + + events := []statemachine.Event{{User: SectorCommitFailed{}}} + + require.NoError(t, planCommitting(events, m.state)) + + require.Equal(t, CommitFailed, m.state.State) +} diff --git a/extern/storage-fsm/garbage.go b/extern/storage-fsm/garbage.go new file mode 100644 index 000000000..0464259c3 --- /dev/null +++ b/extern/storage-fsm/garbage.go @@ -0,0 +1,79 @@ +package sealing + +import ( + "context" + "io" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + + nr "github.com/filecoin-project/storage-fsm/lib/nullreader" +) + +func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader { + return io.LimitReader(&nr.Reader{}, int64(size)) +} + +func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]abi.PieceInfo, error) { + if len(sizes) == 0 { + return nil, nil + } + + log.Infof("Pledge %d, contains %+v", sectorID, existingPieceSizes) + + out := make([]abi.PieceInfo, len(sizes)) + for i, size := range sizes { + ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, m.pledgeReader(size)) + if err != nil { + return nil, xerrors.Errorf("add piece: %w", err) + } + + existingPieceSizes = append(existingPieceSizes, size) + + out[i] = ppi + } + + return out, nil +} + +func (m *Sealing) PledgeSector() error { + go func() { + ctx := context.TODO() // we can't use the context from command which invokes + // this, as we run everything here async, and it's cancelled when the + // command exits + + size := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() + + sid, err := m.sc.Next() + if err != nil { + log.Errorf("%+v", err) + return + } + err = m.sealer.NewSector(ctx, m.minerSector(sid)) + if err != nil { + log.Errorf("%+v", err) + return + } + + pieces, err := m.pledgeSector(ctx, m.minerSector(sid), []abi.UnpaddedPieceSize{}, size) + if err != nil { + log.Errorf("%+v", err) + return + } + + ps := make([]Piece, len(pieces)) + for idx := range ps { + ps[idx] = Piece{ + Piece: pieces[idx], + DealInfo: nil, + } + } + + if err := m.newSectorCC(sid, ps); err != nil { + log.Errorf("%+v", err) + return + } + }() + return nil +} diff --git a/extern/storage-fsm/gen/main.go b/extern/storage-fsm/gen/main.go new file mode 100644 index 000000000..0d5f7507b --- /dev/null +++ b/extern/storage-fsm/gen/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "os" + + gen "github.com/whyrusleeping/cbor-gen" + + sealing "github.com/filecoin-project/storage-fsm" +) + +func main() { + err := gen.WriteMapEncodersToFile("./cbor_gen.go", "sealing", + sealing.Piece{}, + sealing.DealInfo{}, + sealing.DealSchedule{}, + sealing.SectorInfo{}, + sealing.Log{}, + ) + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/extern/storage-fsm/go.mod b/extern/storage-fsm/go.mod new file mode 100644 index 000000000..94fa760b1 --- /dev/null +++ b/extern/storage-fsm/go.mod @@ -0,0 +1,32 @@ +module github.com/filecoin-project/storage-fsm + +go 1.13 + +require ( + github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be + github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 + github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 + github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 + github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 // indirect + github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 + github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15 + github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6 + github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea + github.com/ipfs/go-cid v0.0.6 + github.com/ipfs/go-datastore v0.4.4 + github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242 // indirect + github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 // indirect + github.com/ipfs/go-log/v2 v2.0.3 + github.com/stretchr/testify v1.4.0 + github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d + go.uber.org/zap v1.14.1 // indirect + golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 // indirect + golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d // indirect + golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + gotest.tools v2.2.0+incompatible + honnef.co/go/tools v0.0.1-2020.1.3 // indirect +) + +replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0 diff --git a/extern/storage-fsm/go.sum b/extern/storage-fsm/go.sum new file mode 100644 index 000000000..9375b3c64 --- /dev/null +++ b/extern/storage-fsm/go.sum @@ -0,0 +1,388 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= +github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= +github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be h1:TooKBwR/g8jG0hZ3lqe9S5sy2vTUcLOZLlz3M5wGn2E= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e h1:IOoff6yAZSJ5zHCPY2jzGNwQYQU6ygsRVe/cSnJrY+o= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2 h1:jamfsxfK0Q9yCMHt8MPWx7Aa/O9k2Lve8eSc6FILYGQ= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060 h1:/3qjGMn6ukXgZJHsIbuwGL7ipla8DOV3uHZDBJkBYfU= +github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= +github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhwrN+buDgQCVpZU= +github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.0.3/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= +github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= +github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= +github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= +github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= +github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15 h1:miw6hiusb/MkV1ryoqUKKWnvHhPW00AYtyeCj0L8pqo= +github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo= +github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= +github.com/filecoin-project/specs-actors v0.3.0 h1:QxgAuTrZr5TPqjyprZk0nTYW5o0JWpzbb5v+4UHHvN0= +github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= +github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0= +github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= +github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6 h1:F+GcBdKPdW/wTv6bMJxG9Zj1dc0UGkO6uNOQmKP/g1o= +github.com/filecoin-project/specs-actors v0.7.3-0.20200716231407-60a2ae96d2e6/go.mod h1:JOMUa7EijvpOO4ofD1yeHNmqohkmmnhTvz/IpB6so4c= +github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= +github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= +github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= +github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242 h1:OYVGeYkGSRZdBJ35JHPXQ9deQxlLtJ3Ln0FuaJOu6x8= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200204200533-99b8553ef242/go.mod h1:kq3Pi+UP3oHhAdKexE+kHHYRKMoFNuGero0R7q3hWGg= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-files v0.0.7 h1:s5BRD12ndahqYifeH1S8Z73zqZhR+3IdKYAG9PiETs0= +github.com/ipfs/go-ipfs-files v0.0.7/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= +github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 h1:jIVle1vGSzxyUhseYNEqd7qcDVRrIbJ7UxGwao70cF0= +github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= +github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= +github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log v1.0.3 h1:Gg7SUYSZ7BrqaKMwM+hRgcAkKv4QLfzP4XPQt5Sx/OI= +github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-log/v2 v2.0.3 h1:Q2gXcBoCALyLN/pUQlz1qgu0x3uFV6FzP9oXhpfyJpc= +github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= +github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= +github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= +github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0 h1:c8R11WC8m7KNMkTv/0+Be8vvwo4I3/Ut9AC2FW8fX3U= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e h1:JY8o/ebUUrCYetWmjRCNghxC59cOEaili83rxPRQCLw= +github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d h1:wSxKhvbN7kUoP0sfRS+w2tWr45qlU8409i94hHLOT8w= +github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6 h1:TjszyFsQsyZNHwdVdZ5m7bjmreu0znc2kRYsEml9/Ww= +golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d h1:62ap6LNOjDU6uGmKXHJbSfciMoV+FeI1sRXx/pLDL44= +golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566 h1:OXjomkWHhzUx4+HldlJ2TsMxJdWgEo5CTtspD1wdhdk= +golang.org/x/tools v0.0.0-20200318150045-ba25ddc85566/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= diff --git a/extern/storage-fsm/lib/nullreader/nullreader.go b/extern/storage-fsm/lib/nullreader/nullreader.go new file mode 100644 index 000000000..dc3537ad7 --- /dev/null +++ b/extern/storage-fsm/lib/nullreader/nullreader.go @@ -0,0 +1,11 @@ +package nullreader + +// TODO: extract this to someplace where it can be shared with lotus +type Reader struct{} + +func (Reader) Read(out []byte) (int, error) { + for i := range out { + out[i] = 0 + } + return len(out), nil +} diff --git a/extern/storage-fsm/parameters.json b/extern/storage-fsm/parameters.json new file mode 100644 index 000000000..4ca3e6d2d --- /dev/null +++ b/extern/storage-fsm/parameters.json @@ -0,0 +1,152 @@ +{ + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params": { + "cid": "QmYkygifkXnrnsN4MJsjBFHTQJHx294CyikDgDK8nYxdGh", + "digest": "df3f30442a6d6b4192f5071fb17e820c", + "sector_size": 2048 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk": { + "cid": "QmdXyqbmy2bkJA9Kyhh6z25GrTCq48LwX6c1mxPsm54wi7", + "digest": "0bea3951abf9557a3569f68e52a30c6c", + "sector_size": 2048 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.params": { + "cid": "Qmf5XZZtP5VcYTf65MbKjLVabcS6cYMbr2rFShmfJzh5e5", + "digest": "655e6277638edc8c658094f6f0b33d54", + "sector_size": 536870912 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk": { + "cid": "QmPuhdWnAXBks43emnkqi9FQzyU1gASKyz23zrD27BPGs8", + "digest": "57690e3a6a94c3f704802a674b34f36b", + "sector_size": 536870912 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params": { + "cid": "QmPNVgTN7N5vDtD5u7ERMTLcvUtrKRBfYVUDr6uW3pKhX7", + "digest": "3d390654f58e603b896ac70c653f5676", + "sector_size": 2048 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk": { + "cid": "Qmbj61Zez7v5xA7nSCnmWbyLYznWJDWeusz7Yg8EcgVdoN", + "digest": "8c170a164743c39576a7f47a1b51e6f3", + "sector_size": 2048 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.params": { + "cid": "QmRApb8RZoBK3cqicT7V3ydXg8yVvqPFMPrQNXP33aBihp", + "digest": "b1b58ff9a297b82885e8a7dfb035f83c", + "sector_size": 8388608 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk": { + "cid": "QmcytF1dTdqMFoyXi931j1RgmGtLfR9LLLaBznRt1tPQyD", + "digest": "1a09e00c641f192f55af3433a028f050", + "sector_size": 8388608 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.params": { + "cid": "QmPvr54tWaVeP4WnekivzUAJitTqsQfvikBvAHNEaDNQSw", + "digest": "9380e41368ed4083dbc922b290d3b786", + "sector_size": 8388608 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk": { + "cid": "QmXyVLVDRCcxA9SjT7PeK8HFtyxZ2ZH3SHa8KoGLw8VGJt", + "digest": "f0731a7e20f90704bd38fc5d27882f6d", + "sector_size": 8388608 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.params": { + "cid": "Qmf5f6ko3dqj7qauzXpZqxM9B2x2sL977K6gE2ppNwuJPv", + "digest": "273ebb8c896326b7c292bee8b775fd38", + "sector_size": 536870912 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk": { + "cid": "QmfP3MQe8koW63n5MkDENENVHxib78MJYYyZvbneCsuze8", + "digest": "3dd94da9da64e51b3445bc528d84e76d", + "sector_size": 536870912 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params": { + "cid": "QmYEeeCE8uT2bsVkxcqqUYeMmMEbe6rfmo8wQCv7jFHqqm", + "digest": "c947f2021304ed43b7216f7a8436e294", + "sector_size": 34359738368 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk": { + "cid": "QmXB63ExriFjB4ywWnXTnFwCcLFfCeEP3h15qtL5i7F4aX", + "digest": "ab20d7b253e7e9a0d2ccdf7599ec8ec3", + "sector_size": 34359738368 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params": { + "cid": "QmW5Yxg3L1NSzuQVcRMHMbG3uvVoi4dTLzVaDpnEUPQpnA", + "digest": "079ba19645828ae42b22b0e3f4866e8d", + "sector_size": 34359738368 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk": { + "cid": "QmQzZ5dJ11tcSBees38WX41tZLXS9BqpEti253m5QcnTNs", + "digest": "c76125a50a7de315165de359b5174ae4", + "sector_size": 34359738368 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.params": { + "cid": "QmNk3wga1tS53FUu1QnkK8ehWA2cqpCnSEAPv3KLxdJxNa", + "digest": "421e4790c0b80e0107a7ff67acf14084", + "sector_size": 68719476736 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk": { + "cid": "QmVQCHGsrUtbn9RjHs1e6GXfeXDW5m9w4ge48PSX3Z2as2", + "digest": "8b60e9cc1470a6729c687d6cf0a1f79c", + "sector_size": 68719476736 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.params": { + "cid": "QmTL3VvydaMFWKvE5VzxjgKsJYgL9JMM4JVYNtQxdj9JK1", + "digest": "2685f31124b22ea6b2857e5a5e87ffa3", + "sector_size": 68719476736 + }, + "v26-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk": { + "cid": "QmSVWbLqQYbUbbJyfsRMzEib2rfSqMtnPks1Nw22omcBQm", + "digest": "efe703cd2839597c7ca5c2a906b74296", + "sector_size": 68719476736 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params": { + "cid": "QmU9dH31nZZUJnsogR4Ld4ySUcH6wm2RgmGiujwnqtbU6k", + "digest": "fcef8e87ae2afd7a28aae44347b804cf", + "sector_size": 2048 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk": { + "cid": "QmdJ15DMGPooye5NaPcRfXUdHUDibcN7hKjbmTGuu1K4AQ", + "digest": "2ee2b3518229680db15161d4f582af37", + "sector_size": 2048 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.params": { + "cid": "QmZgtxcY3tMXXQxZTA7ZTUDXLVUnfxNcerXgeW4gG2NnfP", + "digest": "3273c7135cb75684248b475781b738ee", + "sector_size": 536870912 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk": { + "cid": "QmSS6ZkAV2aGZcgKgdPpEEgihXF1ryZX8PSAZDWSoeL1d4", + "digest": "1519b5f61d9044a59f2bdc57537c094b", + "sector_size": 536870912 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.params": { + "cid": "QmQBGXeiNn6hVwbR6qFarQqiNGDdKk4h9ucfyvcXyfYz2N", + "digest": "7d5f896f435c38e93bcda6dd168d860b", + "sector_size": 8388608 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk": { + "cid": "QmPrZgBVGMckEAeu5eSJnLmiAwcPQjKjZe5ir6VaQ5AxKs", + "digest": "fe6d2de44580a0db5a4934688899b92f", + "sector_size": 8388608 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params": { + "cid": "QmZL2cq45XJn5BFzagAZwgFmLrcM1W6CXoiEF9C5j5tjEF", + "digest": "acdfed9f0512bc85a01a9fb871d475d5", + "sector_size": 34359738368 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk": { + "cid": "QmQ4zB7nNa1tDYNifBkExRnZtwtxZw775iaqvVsZyRi6Q2", + "digest": "524a2f3e9d6826593caebc41bb545c40", + "sector_size": 34359738368 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.params": { + "cid": "QmY7DitNKXFeLQt9QoVQkfjM1EvRnprqUVxjmkTXkHDNka", + "digest": "f27271c0537ba65ade2ec045f8fbd069", + "sector_size": 68719476736 + }, + "v26-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk": { + "cid": "QmUJsvoCuQ4LszPmeRVAkMYb5qY95ctz3UXKhu8xLzyFKo", + "digest": "576b292938c6c9d0a0e721bd867a543b", + "sector_size": 68719476736 + } +} \ No newline at end of file diff --git a/extern/storage-fsm/precommit_policy.go b/extern/storage-fsm/precommit_policy.go new file mode 100644 index 000000000..1521dfb05 --- /dev/null +++ b/extern/storage-fsm/precommit_policy.go @@ -0,0 +1,82 @@ +package sealing + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +type PreCommitPolicy interface { + Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) +} + +type Chain interface { + ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) +} + +// BasicPreCommitPolicy satisfies PreCommitPolicy. It has two modes: +// +// Mode 1: The sector contains a non-zero quantity of pieces with deal info +// Mode 2: The sector contains no pieces with deal info +// +// The BasicPreCommitPolicy#Expiration method is given a slice of the pieces +// which the miner has encoded into the sector, and from that slice picks either +// the first or second mode. +// +// If we're in Mode 1: The pre-commit expiration epoch will be the maximum +// deal end epoch of a piece in the sector. +// +// If we're in Mode 2: The pre-commit expiration epoch will be set to the +// current epoch + the provided default duration. +type BasicPreCommitPolicy struct { + api Chain + + provingBoundary abi.ChainEpoch + duration abi.ChainEpoch +} + +// NewBasicPreCommitPolicy produces a BasicPreCommitPolicy +func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch, provingBoundary abi.ChainEpoch) BasicPreCommitPolicy { + return BasicPreCommitPolicy{ + api: api, + provingBoundary: provingBoundary, + duration: duration, + } +} + +// Expiration produces the pre-commit sector expiration epoch for an encoded +// replica containing the provided enumeration of pieces and deals. +func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) { + _, epoch, err := p.api.ChainHead(ctx) + if err != nil { + return 0, nil + } + + var end *abi.ChainEpoch + + for _, p := range ps { + if p.DealInfo == nil { + continue + } + + if p.DealInfo.DealSchedule.EndEpoch < epoch { + log.Warnf("piece schedule %+v ended before current epoch %d", p, epoch) + continue + } + + if end == nil || *end < p.DealInfo.DealSchedule.EndEpoch { + tmp := p.DealInfo.DealSchedule.EndEpoch + end = &tmp + } + } + + if end == nil { + tmp := epoch + p.duration + end = &tmp + } + + *end += miner.WPoStProvingPeriod - (*end % miner.WPoStProvingPeriod) + p.provingBoundary - 1 + + return *end, nil +} diff --git a/extern/storage-fsm/precommit_policy_test.go b/extern/storage-fsm/precommit_policy_test.go new file mode 100644 index 000000000..5dd24150c --- /dev/null +++ b/extern/storage-fsm/precommit_policy_test.go @@ -0,0 +1,134 @@ +package sealing_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/specs-actors/actors/abi" + + sealing "github.com/filecoin-project/storage-fsm" +) + +type fakeChain struct { + h abi.ChainEpoch +} + +func (f *fakeChain) ChainHead(ctx context.Context) (sealing.TipSetToken, abi.ChainEpoch, error) { + return []byte{1, 2, 3}, f.h, nil +} + +func TestBasicPolicyEmptySector(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 10, 0) + + exp, err := policy.Expiration(context.Background()) + require.NoError(t, err) + + assert.Equal(t, 3455, int(exp)) +} + +func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100, 11) + + pieces := []sealing.Piece{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(42), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(70), + EndEpoch: abi.ChainEpoch(75), + }, + }, + }, + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(43), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(80), + EndEpoch: abi.ChainEpoch(100), + }, + }, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 3466, int(exp)) +} + +func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100, 0) + + pieces := []sealing.Piece{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(44), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(1), + EndEpoch: abi.ChainEpoch(10), + }, + }, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 3455, int(exp)) +} + +func TestMissingDealIsIgnored(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100, 11) + + pieces := []sealing.Piece{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(44), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(1), + EndEpoch: abi.ChainEpoch(10), + }, + }, + }, + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: nil, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 3466, int(exp)) +} diff --git a/extern/storage-fsm/sealing.go b/extern/storage-fsm/sealing.go new file mode 100644 index 000000000..d15255696 --- /dev/null +++ b/extern/storage-fsm/sealing.go @@ -0,0 +1,320 @@ +package sealing + +import ( + "context" + "io" + "sync" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + padreader "github.com/filecoin-project/go-padreader" + statemachine "github.com/filecoin-project/go-statemachine" + sectorstorage "github.com/filecoin-project/sector-storage" + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/crypto" +) + +const SectorStorePrefix = "/sectors" + +var log = logging.Logger("sectors") + +type SectorLocation struct { + Deadline uint64 + Partition uint64 +} + +type SealingAPI interface { + StateWaitMsg(context.Context, cid.Cid) (MsgLookup, error) + StateComputeDataCommitment(ctx context.Context, maddr address.Address, sectorType abi.RegisteredSealProof, deals []abi.DealID, tok TipSetToken) (cid.Cid, error) + StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*miner.SectorPreCommitOnChainInfo, error) + StateSectorGetInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*miner.SectorOnChainInfo, error) + StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*SectorLocation, error) + StateMinerSectorSize(context.Context, address.Address, TipSetToken) (abi.SectorSize, error) + StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok TipSetToken) (address.Address, error) + StateMinerDeadlines(ctx context.Context, maddr address.Address, tok TipSetToken) ([]*miner.Deadline, error) + StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) + StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error) + StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error) + SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, gasPrice big.Int, gasLimit int64, params []byte) (cid.Cid, error) + ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) + ChainGetRandomness(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) +} + +type Sealing struct { + api SealingAPI + events Events + + maddr address.Address + + sealer sectorstorage.SectorManager + sectors *statemachine.StateGroup + sc SectorIDCounter + verif ffiwrapper.Verifier + + pcp PreCommitPolicy + unsealedInfoMap UnsealedSectorMap + + upgradeLk sync.Mutex + toUpgrade map[abi.SectorNumber]struct{} + + getSealDelay GetSealingDelayFunc +} + +type UnsealedSectorMap struct { + infos map[abi.SectorNumber]UnsealedSectorInfo + mux sync.Mutex +} + +type UnsealedSectorInfo struct { + numDeals uint64 + // stored should always equal sum of pieceSizes.Padded() + stored abi.PaddedPieceSize + pieceSizes []abi.UnpaddedPieceSize +} + +func New(api SealingAPI, events Events, maddr address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, pcp PreCommitPolicy, gsd GetSealingDelayFunc) *Sealing { + s := &Sealing{ + api: api, + events: events, + + maddr: maddr, + sealer: sealer, + sc: sc, + verif: verif, + pcp: pcp, + unsealedInfoMap: UnsealedSectorMap{ + infos: make(map[abi.SectorNumber]UnsealedSectorInfo), + mux: sync.Mutex{}, + }, + + toUpgrade: map[abi.SectorNumber]struct{}{}, + getSealDelay: gsd, + } + + s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{}) + + return s +} + +func (m *Sealing) Run(ctx context.Context) error { + if err := m.restartSectors(ctx); err != nil { + log.Errorf("%+v", err) + return xerrors.Errorf("failed load sector states: %w", err) + } + + return nil +} + +func (m *Sealing) Stop(ctx context.Context) error { + return m.sectors.Stop(ctx) +} +func (m *Sealing) AddPieceToAnySector(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, d DealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) { + log.Infof("Adding piece for deal %d", d.DealID) + if (padreader.PaddedSize(uint64(size))) != size { + return 0, 0, xerrors.Errorf("cannot allocate unpadded piece") + } + + if size > abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() { + return 0, 0, xerrors.Errorf("piece cannot fit into a sector") + } + + m.unsealedInfoMap.mux.Lock() + + sid, pads, err := m.getSectorAndPadding(size) + if err != nil { + m.unsealedInfoMap.mux.Unlock() + return 0, 0, xerrors.Errorf("getting available sector: %w", err) + } + + for _, p := range pads { + err = m.addPiece(ctx, sid, p.Unpadded(), m.pledgeReader(p.Unpadded()), nil) + if err != nil { + m.unsealedInfoMap.mux.Unlock() + return 0, 0, xerrors.Errorf("writing pads: %w", err) + } + } + + offset := m.unsealedInfoMap.infos[sid].stored + err = m.addPiece(ctx, sid, size, r, &d) + + if err != nil { + m.unsealedInfoMap.mux.Unlock() + return 0, 0, xerrors.Errorf("adding piece to sector: %w", err) + } + + m.unsealedInfoMap.mux.Unlock() + if m.unsealedInfoMap.infos[sid].numDeals == getDealPerSectorLimit(m.sealer.SectorSize()) { + if err := m.StartPacking(sid); err != nil { + return 0, 0, xerrors.Errorf("start packing: %w", err) + } + } + + return sid, offset, nil +} + +// Caller should hold m.unsealedInfoMap.mux +func (m *Sealing) addPiece(ctx context.Context, sectorID abi.SectorNumber, size abi.UnpaddedPieceSize, r io.Reader, di *DealInfo) error { + log.Infof("Adding piece to sector %d", sectorID) + ppi, err := m.sealer.AddPiece(sectorstorage.WithPriority(ctx, DealSectorPriority), m.minerSector(sectorID), m.unsealedInfoMap.infos[sectorID].pieceSizes, size, r) + if err != nil { + return xerrors.Errorf("writing piece: %w", err) + } + piece := Piece{ + Piece: ppi, + DealInfo: di, + } + + err = m.sectors.Send(uint64(sectorID), SectorAddPiece{NewPiece: piece}) + if err != nil { + return err + } + + ui := m.unsealedInfoMap.infos[sectorID] + num := m.unsealedInfoMap.infos[sectorID].numDeals + if di != nil { + num = num + 1 + } + m.unsealedInfoMap.infos[sectorID] = UnsealedSectorInfo{ + numDeals: num, + stored: ui.stored + piece.Piece.Size, + pieceSizes: append(ui.pieceSizes, piece.Piece.Size.Unpadded()), + } + + return nil +} + +func (m *Sealing) Remove(ctx context.Context, sid abi.SectorNumber) error { + return m.sectors.Send(uint64(sid), SectorRemove{}) +} + +// Caller should NOT hold m.unsealedInfoMap.mux +func (m *Sealing) StartPacking(sectorID abi.SectorNumber) error { + log.Infof("Starting packing sector %d", sectorID) + err := m.sectors.Send(uint64(sectorID), SectorStartPacking{}) + if err != nil { + return err + } + + m.unsealedInfoMap.mux.Lock() + delete(m.unsealedInfoMap.infos, sectorID) + m.unsealedInfoMap.mux.Unlock() + + return nil +} + +// Caller should hold m.unsealedInfoMap.mux +func (m *Sealing) getSectorAndPadding(size abi.UnpaddedPieceSize) (abi.SectorNumber, []abi.PaddedPieceSize, error) { + ss := abi.PaddedPieceSize(m.sealer.SectorSize()) + for k, v := range m.unsealedInfoMap.infos { + pads, padLength := ffiwrapper.GetRequiredPadding(v.stored, size.Padded()) + if v.stored+size.Padded()+padLength <= ss { + return k, pads, nil + } + } + + ns, err := m.newSector() + if err != nil { + return 0, nil, err + } + + m.unsealedInfoMap.infos[ns] = UnsealedSectorInfo{ + numDeals: 0, + stored: 0, + pieceSizes: nil, + } + + return ns, nil, nil +} + +// newSector creates a new sector for deal storage +func (m *Sealing) newSector() (abi.SectorNumber, error) { + sid, err := m.sc.Next() + if err != nil { + return 0, xerrors.Errorf("getting sector number: %w", err) + } + + err = m.sealer.NewSector(context.TODO(), m.minerSector(sid)) + if err != nil { + return 0, xerrors.Errorf("initializing sector: %w", err) + } + + rt, err := ffiwrapper.SealProofTypeFromSectorSize(m.sealer.SectorSize()) + if err != nil { + return 0, xerrors.Errorf("bad sector size: %w", err) + } + + log.Infof("Creating sector %d", sid) + err = m.sectors.Send(uint64(sid), SectorStart{ + ID: sid, + SectorType: rt, + }) + + if err != nil { + return 0, xerrors.Errorf("starting the sector fsm: %w", err) + } + + sd, err := m.getSealDelay() + if err != nil { + return 0, xerrors.Errorf("getting the sealing delay: %w", err) + } + + if sd > 0 { + timer := time.NewTimer(sd) + go func() { + <-timer.C + m.StartPacking(sid) + }() + } + + return sid, nil +} + +// newSectorCC accepts a slice of pieces with no deal (junk data) +func (m *Sealing) newSectorCC(sid abi.SectorNumber, pieces []Piece) error { + rt, err := ffiwrapper.SealProofTypeFromSectorSize(m.sealer.SectorSize()) + if err != nil { + return xerrors.Errorf("bad sector size: %w", err) + } + + log.Infof("Creating CC sector %d", sid) + return m.sectors.Send(uint64(sid), SectorStartCC{ + ID: sid, + Pieces: pieces, + SectorType: rt, + }) +} + +func (m *Sealing) minerSector(num abi.SectorNumber) abi.SectorID { + mid, err := address.IDFromAddress(m.maddr) + if err != nil { + panic(err) + } + + return abi.SectorID{ + Number: num, + Miner: abi.ActorID(mid), + } +} + +func (m *Sealing) Address() address.Address { + return m.maddr +} + +func getDealPerSectorLimit(size abi.SectorSize) uint64 { + if size < 64<<30 { + return 256 + } else { + return 512 + } +} diff --git a/extern/storage-fsm/sector_state.go b/extern/storage-fsm/sector_state.go new file mode 100644 index 000000000..f2801c9fc --- /dev/null +++ b/extern/storage-fsm/sector_state.go @@ -0,0 +1,38 @@ +package sealing + +type SectorState string + +const ( + UndefinedSectorState SectorState = "" + + // happy path + Empty SectorState = "Empty" + WaitDeals SectorState = "WaitDeals" // waiting for more pieces (deals) to be added to the sector + Packing SectorState = "Packing" // sector not in sealStore, and not on chain + PreCommit1 SectorState = "PreCommit1" // do PreCommit1 + PreCommit2 SectorState = "PreCommit2" // do PreCommit1 + PreCommitting SectorState = "PreCommitting" // on chain pre-commit + PreCommitWait SectorState = "PreCommitWait" // waiting for precommit to land on chain + WaitSeed SectorState = "WaitSeed" // waiting for seed + Committing SectorState = "Committing" + CommitWait SectorState = "CommitWait" // waiting for message to land on chain + FinalizeSector SectorState = "FinalizeSector" + Proving SectorState = "Proving" + // error modes + FailedUnrecoverable SectorState = "FailedUnrecoverable" + SealPreCommit1Failed SectorState = "SealPreCommit1Failed" + SealPreCommit2Failed SectorState = "SealPreCommit2Failed" + PreCommitFailed SectorState = "PreCommitFailed" + ComputeProofFailed SectorState = "ComputeProofFailed" + CommitFailed SectorState = "CommitFailed" + PackingFailed SectorState = "PackingFailed" + FinalizeFailed SectorState = "FinalizeFailed" + + Faulty SectorState = "Faulty" // sector is corrupted or gone for some reason + FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain + FaultedFinal SectorState = "FaultedFinal" // fault declared on chain + + Removing SectorState = "Removing" + RemoveFailed SectorState = "RemoveFailed" + Removed SectorState = "Removed" +) diff --git a/extern/storage-fsm/states_failed.go b/extern/storage-fsm/states_failed.go new file mode 100644 index 000000000..e208a8cca --- /dev/null +++ b/extern/storage-fsm/states_failed.go @@ -0,0 +1,211 @@ +package sealing + +import ( + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +const minRetryTime = 1 * time.Minute + +func failedCooldown(ctx statemachine.Context, sector SectorInfo) error { + // TODO: Exponential backoff when we see consecutive failures + + retryStart := time.Unix(int64(sector.Log[len(sector.Log)-1].Timestamp), 0).Add(minRetryTime) + if len(sector.Log) > 0 && !time.Now().After(retryStart) { + log.Infof("%s(%d), waiting %s before retrying", sector.State, sector.SectorNumber, time.Until(retryStart)) + select { + case <-time.After(time.Until(retryStart)): + case <-ctx.Context().Done(): + return ctx.Context().Err() + } + } + + return nil +} + +func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) (*miner.SectorPreCommitOnChainInfo, bool) { + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) + return nil, true + } + + info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) + if err != nil { + log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err) + return nil, true + } + + return info, false +} + +func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error { + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetrySealPreCommit1{}) +} + +func (m *Sealing) handleSealPrecommit2Failed(ctx statemachine.Context, sector SectorInfo) error { + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + if sector.PreCommit2Fails > 1 { + return ctx.Send(SectorRetrySealPreCommit1{}) + } + + return ctx.Send(SectorRetrySealPreCommit2{}) +} + +func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorInfo) error { + tok, height, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err) + return nil + } + + if err := checkPrecommit(ctx.Context(), m.Address(), sector, tok, height, m.api); err != nil { + switch err.(type) { + case *ErrApi: + log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err) + return nil + case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) + case *ErrExpiredTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) + case *ErrNoPrecommit: + return ctx.Send(SectorRetryPreCommit{}) + case *ErrPrecommitOnChain: + // noop + default: + return xerrors.Errorf("checkPrecommit sanity check error: %w", err) + } + } + + if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil { + if sector.PreCommitMessage != nil { + log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber) + return ctx.Send(SectorPreCommitLanded{TipSet: tok}) + } + + if pci.Info.SealedCID != *sector.CommR { + log.Warn("sector %d is precommitted on chain, with different CommR: %x != %x", sector.SectorNumber, pci.Info.SealedCID, sector.CommR) + return nil // TODO: remove when the actor allows re-precommit + } + + // TODO: we could compare more things, but I don't think we really need to + // CommR tells us that CommD (and CommPs), and the ticket are all matching + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetryWaitSeed{}) + } + + if sector.PreCommitMessage != nil { + log.Warn("retrying precommit even though the message failed to apply") + } + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetryPreCommit{}) +} + +func (m *Sealing) handleComputeProofFailed(ctx statemachine.Context, sector SectorInfo) error { + // TODO: Check sector files + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + if sector.InvalidProofs > 1 { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive compute fails")}) + } + + return ctx.Send(SectorRetryComputeProof{}) +} + +func (m *Sealing) handleCommitFailed(ctx statemachine.Context, sector SectorInfo) error { + tok, height, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + if err := checkPrecommit(ctx.Context(), m.maddr, sector, tok, height, m.api); err != nil { + switch err.(type) { + case *ErrApi: + log.Errorf("handleCommitFailed: api error, not proceeding: %+v", err) + return nil + case *ErrBadCommD: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) + case *ErrExpiredTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired error: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad expired: %w", err)}) + case nil: + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("no precommit: %w", err)}) + case *ErrPrecommitOnChain: + // noop, this is expected + default: + return xerrors.Errorf("checkPrecommit sanity check error (%T): %w", err, err) + } + } + + if err := m.checkCommit(ctx.Context(), sector, sector.Proof, tok); err != nil { + switch err.(type) { + case *ErrApi: + log.Errorf("handleCommitFailed: api error, not proceeding: %+v", err) + return nil + case *ErrBadSeed: + log.Errorf("seed changed, will retry: %+v", err) + return ctx.Send(SectorRetryWaitSeed{}) + case *ErrInvalidProof: + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + if sector.InvalidProofs > 0 { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("consecutive invalid proofs")}) + } + + return ctx.Send(SectorRetryInvalidProof{}) + case *ErrPrecommitOnChain: + log.Errorf("no precommit on chain, will retry: %+v", err) + return ctx.Send(SectorRetryPreCommitWait{}) + case *ErrNoPrecommit: + return ctx.Send(SectorRetryPreCommit{}) + default: + return xerrors.Errorf("checkCommit sanity check error (%T): %w", err, err) + } + } + + // TODO: Check sector files + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetryComputeProof{}) +} + +func (m *Sealing) handleFinalizeFailed(ctx statemachine.Context, sector SectorInfo) error { + // TODO: Check sector files + + if err := failedCooldown(ctx, sector); err != nil { + return err + } + + return ctx.Send(SectorRetryFinalize{}) +} diff --git a/extern/storage-fsm/states_proving.go b/extern/storage-fsm/states_proving.go new file mode 100644 index 000000000..6684c714d --- /dev/null +++ b/extern/storage-fsm/states_proving.go @@ -0,0 +1,40 @@ +package sealing + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-statemachine" +) + +func (m *Sealing) handleFaulty(ctx statemachine.Context, sector SectorInfo) error { + // TODO: noop because this is now handled by the PoSt scheduler. We can reuse + // this state for tracking faulty sectors, or remove it when that won't be + // a breaking change + return nil +} + +func (m *Sealing) handleFaultReported(ctx statemachine.Context, sector SectorInfo) error { + if sector.FaultReportMsg == nil { + return xerrors.Errorf("entered fault reported state without a FaultReportMsg cid") + } + + mw, err := m.api.StateWaitMsg(ctx.Context(), *sector.FaultReportMsg) + if err != nil { + return xerrors.Errorf("failed to wait for fault declaration: %w", err) + } + + if mw.Receipt.ExitCode != 0 { + log.Errorf("UNHANDLED: declaring sector fault failed (exit=%d, msg=%s) (id: %d)", mw.Receipt.ExitCode, *sector.FaultReportMsg, sector.SectorNumber) + return xerrors.Errorf("UNHANDLED: submitting fault declaration failed (exit %d)", mw.Receipt.ExitCode) + } + + return ctx.Send(SectorFaultedFinal{}) +} + +func (m *Sealing) handleRemoving(ctx statemachine.Context, sector SectorInfo) error { + if err := m.sealer.Remove(ctx.Context(), m.minerSector(sector.SectorNumber)); err != nil { + return ctx.Send(SectorRemoveFailed{err}) + } + + return ctx.Send(SectorRemoved{}) +} diff --git a/extern/storage-fsm/states_sealing.go b/extern/storage-fsm/states_sealing.go new file mode 100644 index 000000000..11e8a39a5 --- /dev/null +++ b/extern/storage-fsm/states_sealing.go @@ -0,0 +1,392 @@ +package sealing + +import ( + "bytes" + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-statemachine" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/specs-storage/storage" +) + +var DealSectorPriority = 1024 + +func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) error { + log.Infow("performing filling up rest of the sector...", "sector", sector.SectorNumber) + + var allocated abi.UnpaddedPieceSize + for _, piece := range sector.Pieces { + allocated += piece.Piece.Size.Unpadded() + } + + ubytes := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() + + if allocated > ubytes { + return xerrors.Errorf("too much data in sector: %d > %d", allocated, ubytes) + } + + fillerSizes, err := fillersFromRem(ubytes - allocated) + if err != nil { + return err + } + + if len(fillerSizes) > 0 { + log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorNumber) + } + + fillerPieces, err := m.pledgeSector(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), sector.existingPieceSizes(), fillerSizes...) + if err != nil { + return xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) + } + + return ctx.Send(SectorPacked{FillerPieces: fillerPieces}) +} + +func (m *Sealing) getTicket(ctx statemachine.Context, sector SectorInfo) (abi.SealRandomness, abi.ChainEpoch, error) { + tok, epoch, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err) + return nil, 0, nil + } + + ticketEpoch := epoch - SealRandomnessLookback + buf := new(bytes.Buffer) + if err := m.maddr.MarshalCBOR(buf); err != nil { + return nil, 0, err + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) + if err != nil { + return nil, 0, xerrors.Errorf("getting precommit info: %w", err) + } + + if pci != nil { + ticketEpoch = pci.Info.SealRandEpoch + } + + rand, err := m.api.ChainGetRandomness(ctx.Context(), tok, crypto.DomainSeparationTag_SealRandomness, ticketEpoch, buf.Bytes()) + if err != nil { + return nil, 0, err + } + + return abi.SealRandomness(rand), ticketEpoch, nil +} + +func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error { + if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state + switch err.(type) { + case *ErrApi: + log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err) + return nil + case *ErrInvalidDeals: + return ctx.Send(SectorPackingFailed{xerrors.Errorf("invalid dealIDs in sector: %w", err)}) + case *ErrExpiredDeals: // Probably not much we can do here, maybe re-pack the sector? + return ctx.Send(SectorPackingFailed{xerrors.Errorf("expired dealIDs in sector: %w", err)}) + default: + return xerrors.Errorf("checkPieces sanity check error: %w", err) + } + } + + log.Infow("performing sector replication...", "sector", sector.SectorNumber) + ticketValue, ticketEpoch, err := m.getTicket(ctx, sector) + if err != nil { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("getting ticket failed: %w", err)}) + } + + pc1o, err := m.sealer.SealPreCommit1(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), ticketValue, sector.pieceInfos()) + if err != nil { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("seal pre commit(1) failed: %w", err)}) + } + + return ctx.Send(SectorPreCommit1{ + PreCommit1Out: pc1o, + TicketValue: ticketValue, + TicketEpoch: ticketEpoch, + }) +} + +func (m *Sealing) handlePreCommit2(ctx statemachine.Context, sector SectorInfo) error { + cids, err := m.sealer.SealPreCommit2(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), sector.PreCommit1Out) + if err != nil { + return ctx.Send(SectorSealPreCommit2Failed{xerrors.Errorf("seal pre commit(2) failed: %w", err)}) + } + + return ctx.Send(SectorPreCommit2{ + Unsealed: cids.Unsealed, + Sealed: cids.Sealed, + }) +} + +func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInfo) error { + tok, height, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err) + return nil + } + + waddr, err := m.api.StateMinerWorkerAddress(ctx.Context(), m.maddr, tok) + if err != nil { + log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err) + return nil + } + + if err := checkPrecommit(ctx.Context(), m.Address(), sector, tok, height, m.api); err != nil { + switch err := err.(type) { + case *ErrApi: + log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err) + return nil + case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handlePreCommit1 will do that too) + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad CommD error: %w", err)}) + case *ErrExpiredTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("ticket expired: %w", err)}) + case *ErrBadTicket: + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("bad ticket: %w", err)}) + case *ErrPrecommitOnChain: + return ctx.Send(SectorPreCommitLanded{TipSet: tok}) // we re-did precommit + default: + return xerrors.Errorf("checkPrecommit sanity check error: %w", err) + } + } + + expiration, err := m.pcp.Expiration(ctx.Context(), sector.Pieces...) + if err != nil { + return ctx.Send(SectorSealPreCommit1Failed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)}) + } + + params := &miner.SectorPreCommitInfo{ + Expiration: expiration, + SectorNumber: sector.SectorNumber, + SealProof: sector.SectorType, + + SealedCID: *sector.CommR, + SealRandEpoch: sector.TicketEpoch, + DealIDs: sector.dealIDs(), + } + + depositMinimum := m.tryUpgradeSector(ctx.Context(), params) + + enc := new(bytes.Buffer) + if err := params.MarshalCBOR(enc); err != nil { + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("could not serialize pre-commit sector parameters: %w", err)}) + } + + collateral, err := m.api.StateMinerPreCommitDepositForPower(ctx.Context(), m.maddr, *params, tok) + if err != nil { + return xerrors.Errorf("getting initial pledge collateral: %w", err) + } + + deposit := big.Max(depositMinimum, collateral) + + log.Infof("submitting precommit for sector %d (deposit: %s): ", sector.SectorNumber, deposit) + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.PreCommitSector, deposit, big.NewInt(0), 0, enc.Bytes()) + if err != nil { + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) + } + + return ctx.Send(SectorPreCommitted{Message: mcid, PreCommitDeposit: deposit, PreCommitInfo: *params}) +} + +func (m *Sealing) handlePreCommitWait(ctx statemachine.Context, sector SectorInfo) error { + if sector.PreCommitMessage == nil { + return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("precommit message was nil")}) + } + + // would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts + log.Info("Sector precommitted: ", sector.SectorNumber) + mw, err := m.api.StateWaitMsg(ctx.Context(), *sector.PreCommitMessage) + if err != nil { + return ctx.Send(SectorChainPreCommitFailed{err}) + } + + if mw.Receipt.ExitCode != 0 { + log.Error("sector precommit failed: ", mw.Receipt.ExitCode) + err := xerrors.Errorf("sector precommit failed: %d", mw.Receipt.ExitCode) + return ctx.Send(SectorChainPreCommitFailed{err}) + } + log.Info("precommit message landed on chain: ", sector.SectorNumber) + + return ctx.Send(SectorPreCommitLanded{TipSet: mw.TipSetTok}) +} + +func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error { + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) + if err != nil { + return xerrors.Errorf("getting precommit info: %w", err) + } + if pci == nil { + return ctx.Send(SectorChainPreCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) + } + + randHeight := pci.PreCommitEpoch + miner.PreCommitChallengeDelay + + err = m.events.ChainAt(func(ectx context.Context, _ TipSetToken, curH abi.ChainEpoch) error { + // in case of null blocks the randomness can land after the tipset we + // get from the events API + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + buf := new(bytes.Buffer) + if err := m.maddr.MarshalCBOR(buf); err != nil { + return err + } + rand, err := m.api.ChainGetRandomness(ectx, tok, crypto.DomainSeparationTag_InteractiveSealChallengeSeed, randHeight, buf.Bytes()) + if err != nil { + err = xerrors.Errorf("failed to get randomness for computing seal proof (ch %d; rh %d; tsk %x): %w", curH, randHeight, tok, err) + + _ = ctx.Send(SectorChainPreCommitFailed{error: err}) + return err + } + + _ = ctx.Send(SectorSeedReady{SeedValue: abi.InteractiveSealRandomness(rand), SeedEpoch: randHeight}) + + return nil + }, func(ctx context.Context, ts TipSetToken) error { + log.Warn("revert in interactive commit sector step") + // TODO: need to cancel running process and restart... + return nil + }, InteractivePoRepConfidence, randHeight) + if err != nil { + log.Warn("waitForPreCommitMessage ChainAt errored: ", err) + } + + return nil +} + +func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) error { + log.Info("scheduling seal proof computation...") + + log.Infof("KOMIT %d %x(%d); %x(%d); %v; r:%x; d:%x", sector.SectorNumber, sector.TicketValue, sector.TicketEpoch, sector.SeedValue, sector.SeedEpoch, sector.pieceInfos(), sector.CommR, sector.CommD) + + cids := storage.SectorCids{ + Unsealed: *sector.CommD, + Sealed: *sector.CommR, + } + c2in, err := m.sealer.SealCommit1(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), sector.TicketValue, sector.SeedValue, sector.pieceInfos(), cids) + if err != nil { + return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(1): %w", err)}) + } + + proof, err := m.sealer.SealCommit2(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), c2in) + if err != nil { + return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed(2): %w", err)}) + } + + tok, _, err := m.api.ChainHead(ctx.Context()) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + if err := m.checkCommit(ctx.Context(), sector, proof, tok); err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("commit check error: %w", err)}) + } + + // TODO: Consider splitting states and persist proof for faster recovery + + params := &miner.ProveCommitSectorParams{ + SectorNumber: sector.SectorNumber, + Proof: proof, + } + + enc := new(bytes.Buffer) + if err := params.MarshalCBOR(enc); err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("could not serialize commit sector parameters: %w", err)}) + } + + waddr, err := m.api.StateMinerWorkerAddress(ctx.Context(), m.maddr, tok) + if err != nil { + log.Errorf("handleCommitting: api error, not proceeding: %+v", err) + return nil + } + + collateral, err := m.api.StateMinerInitialPledgeCollateral(ctx.Context(), m.maddr, *sector.PreCommitInfo, tok) + if err != nil { + return xerrors.Errorf("getting initial pledge collateral: %w", err) + } + + pci, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok) + if err != nil { + return xerrors.Errorf("getting precommit info: %w", err) + } + if pci == nil { + return ctx.Send(SectorCommitFailed{error: xerrors.Errorf("precommit info not found on chain")}) + } + + collateral = big.Sub(collateral, pci.PreCommitDeposit) + if collateral.LessThan(big.Zero()) { + collateral = big.Zero() + } + + // TODO: check seed / ticket are up to date + mcid, err := m.api.SendMsg(ctx.Context(), waddr, m.maddr, builtin.MethodsMiner.ProveCommitSector, collateral, big.NewInt(0), 0, enc.Bytes()) + if err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) + } + + return ctx.Send(SectorCommitted{ + Proof: proof, + Message: mcid, + }) +} + +func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo) error { + if sector.CommitMessage == nil { + log.Errorf("sector %d entered commit wait state without a message cid", sector.SectorNumber) + return ctx.Send(SectorCommitFailed{xerrors.Errorf("entered commit wait with no commit cid")}) + } + + mw, err := m.api.StateWaitMsg(ctx.Context(), *sector.CommitMessage) + if err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("failed to wait for porep inclusion: %w", err)}) + } + + if mw.Receipt.ExitCode != 0 { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.TicketValue, sector.SeedValue, sector.SeedEpoch, sector.Proof)}) + } + + _, err = m.api.StateSectorGetInfo(ctx.Context(), m.maddr, sector.SectorNumber, mw.TipSetTok) + if err != nil { + return ctx.Send(SectorCommitFailed{xerrors.Errorf("proof validation failed, sector not found in sector set after cron: %w", err)}) + } + + return ctx.Send(SectorProving{}) +} + +func (m *Sealing) handleFinalizeSector(ctx statemachine.Context, sector SectorInfo) error { + // TODO: Maybe wait for some finality + + if err := m.sealer.FinalizeSector(sector.sealingCtx(ctx.Context()), m.minerSector(sector.SectorNumber), sector.keepUnsealedRanges(false)); err != nil { + return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("finalize sector: %w", err)}) + } + + return ctx.Send(SectorFinalized{}) +} + +func (m *Sealing) handleProvingSector(ctx statemachine.Context, sector SectorInfo) error { + // TODO: track sector health / expiration + log.Infof("Proving sector %d", sector.SectorNumber) + + if err := m.sealer.ReleaseUnsealed(ctx.Context(), m.minerSector(sector.SectorNumber), sector.keepUnsealedRanges(true)); err != nil { + log.Error(err) + } + + // TODO: Watch termination + // TODO: Auto-extend if set + + return nil +} diff --git a/extern/storage-fsm/types.go b/extern/storage-fsm/types.go new file mode 100644 index 000000000..08bdcb739 --- /dev/null +++ b/extern/storage-fsm/types.go @@ -0,0 +1,194 @@ +package sealing + +import ( + "bytes" + "context" + "time" + + "github.com/ipfs/go-cid" + + sectorstorage "github.com/filecoin-project/sector-storage" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/runtime/exitcode" + "github.com/filecoin-project/specs-storage/storage" +) + +// Piece is a tuple of piece and deal info +type PieceWithDealInfo struct { + Piece abi.PieceInfo + DealInfo DealInfo +} + +// Piece is a tuple of piece info and optional deal +type Piece struct { + Piece abi.PieceInfo + DealInfo *DealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) +} + +// DealInfo is a tuple of deal identity and its schedule +type DealInfo struct { + DealID abi.DealID + DealSchedule DealSchedule + KeepUnsealed bool +} + +// DealSchedule communicates the time interval of a storage deal. The deal must +// appear in a sealed (proven) sector no later than StartEpoch, otherwise it +// is invalid. +type DealSchedule struct { + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch +} + +type Log struct { + Timestamp uint64 + Trace string // for errors + + Message string + + // additional data (Event info) + Kind string +} + +type SectorInfo struct { + State SectorState + SectorNumber abi.SectorNumber + + SectorType abi.RegisteredSealProof + + // Packing + Pieces []Piece + + // PreCommit1 + TicketValue abi.SealRandomness + TicketEpoch abi.ChainEpoch + PreCommit1Out storage.PreCommit1Out + + // PreCommit2 + CommD *cid.Cid + CommR *cid.Cid + Proof []byte + + PreCommitInfo *miner.SectorPreCommitInfo + PreCommitDeposit big.Int + PreCommitMessage *cid.Cid + PreCommitTipSet TipSetToken + + PreCommit2Fails uint64 + + // WaitSeed + SeedValue abi.InteractiveSealRandomness + SeedEpoch abi.ChainEpoch + + // Committing + CommitMessage *cid.Cid + InvalidProofs uint64 // failed proof computations (doesn't validate with proof inputs; can't compute) + + // Faults + FaultReportMsg *cid.Cid + + // Debug + LastErr string + + Log []Log +} + +func (t *SectorInfo) pieceInfos() []abi.PieceInfo { + out := make([]abi.PieceInfo, len(t.Pieces)) + for i, p := range t.Pieces { + out[i] = p.Piece + } + return out +} + +func (t *SectorInfo) dealIDs() []abi.DealID { + out := make([]abi.DealID, 0, len(t.Pieces)) + for _, p := range t.Pieces { + if p.DealInfo == nil { + continue + } + out = append(out, p.DealInfo.DealID) + } + return out +} + +func (t *SectorInfo) existingPieceSizes() []abi.UnpaddedPieceSize { + out := make([]abi.UnpaddedPieceSize, len(t.Pieces)) + for i, p := range t.Pieces { + out[i] = p.Piece.Size.Unpadded() + } + return out +} + +func (t *SectorInfo) hasDeals() bool { + for _, piece := range t.Pieces { + if piece.DealInfo != nil { + return true + } + } + + return false +} + +func (t *SectorInfo) sealingCtx(ctx context.Context) context.Context { + // TODO: can also take start epoch into account to give priority to sectors + // we need sealed sooner + + if t.hasDeals() { + return sectorstorage.WithPriority(ctx, DealSectorPriority) + } + + return ctx +} + +// Returns list of offset/length tuples of sector data ranges which clients +// requested to keep unsealed +func (t *SectorInfo) keepUnsealedRanges(invert bool) []storage.Range { + var out []storage.Range + + var at abi.UnpaddedPieceSize + for _, piece := range t.Pieces { + psize := piece.Piece.Size.Unpadded() + at += psize + + if piece.DealInfo == nil { + continue + } + if piece.DealInfo.KeepUnsealed == invert { + continue + } + + out = append(out, storage.Range{ + Offset: at - psize, + Size: psize, + }) + } + + return out +} + +type SectorIDCounter interface { + Next() (abi.SectorNumber, error) +} + +type TipSetToken []byte + +type MsgLookup struct { + Receipt MessageReceipt + TipSetTok TipSetToken + Height abi.ChainEpoch +} + +type MessageReceipt struct { + ExitCode exitcode.ExitCode + Return []byte + GasUsed int64 +} + +type GetSealingDelayFunc func() (time.Duration, error) + +func (mr *MessageReceipt) Equals(o *MessageReceipt) bool { + return mr.ExitCode == o.ExitCode && bytes.Equal(mr.Return, o.Return) && mr.GasUsed == o.GasUsed +} diff --git a/extern/storage-fsm/types_test.go b/extern/storage-fsm/types_test.go new file mode 100644 index 000000000..c11cc66b7 --- /dev/null +++ b/extern/storage-fsm/types_test.go @@ -0,0 +1,70 @@ +package sealing + +import ( + "bytes" + "testing" + + "gotest.tools/assert" + + cborutil "github.com/filecoin-project/go-cbor-util" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin" +) + +func TestSectorInfoSelialization(t *testing.T) { + d := abi.DealID(1234) + + dealInfo := DealInfo{ + DealID: d, + DealSchedule: DealSchedule{ + StartEpoch: 0, + EndEpoch: 100, + }, + } + + dummyCid := builtin.AccountActorCodeID + + si := &SectorInfo{ + State: "stateful", + SectorNumber: 234, + Pieces: []Piece{{ + Piece: abi.PieceInfo{ + Size: 5, + PieceCID: dummyCid, + }, + DealInfo: &dealInfo, + }}, + CommD: &dummyCid, + CommR: nil, + Proof: nil, + TicketValue: []byte{87, 78, 7, 87}, + TicketEpoch: 345, + PreCommitMessage: nil, + SeedValue: []byte{}, + SeedEpoch: 0, + CommitMessage: nil, + FaultReportMsg: nil, + LastErr: "hi", + } + + b, err := cborutil.Dump(si) + if err != nil { + t.Fatal(err) + } + + var si2 SectorInfo + if err := cborutil.ReadCborRPC(bytes.NewReader(b), &si); err != nil { + return + } + + assert.Equal(t, si.State, si2.State) + assert.Equal(t, si.SectorNumber, si2.SectorNumber) + + assert.Equal(t, si.Pieces, si2.Pieces) + assert.Equal(t, si.CommD, si2.CommD) + assert.Equal(t, si.TicketValue, si2.TicketValue) + assert.Equal(t, si.TicketEpoch, si2.TicketEpoch) + + assert.Equal(t, si, si2) + +} diff --git a/extern/storage-fsm/upgrade_queue.go b/extern/storage-fsm/upgrade_queue.go new file mode 100644 index 000000000..ed60e55d4 --- /dev/null +++ b/extern/storage-fsm/upgrade_queue.go @@ -0,0 +1,92 @@ +package sealing + +import ( + "context" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" + "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func (m *Sealing) MarkForUpgrade(id abi.SectorNumber) error { + m.upgradeLk.Lock() + defer m.upgradeLk.Unlock() + + _, found := m.toUpgrade[id] + if found { + return xerrors.Errorf("sector %d already marked for upgrade", id) + } + + si, err := m.GetSectorInfo(id) + if err != nil { + return xerrors.Errorf("getting sector info: %w", err) + } + + if si.State != Proving { + return xerrors.Errorf("can't mark sectors not in the 'Proving' state for upgrade") + } + + if len(si.Pieces) != 1 { + return xerrors.Errorf("not a committed-capacity sector, expected 1 piece") + } + + if si.Pieces[0].DealInfo != nil { + return xerrors.Errorf("not a committed-capacity sector, has deals") + } + + // TODO: more checks to match actor constraints + + m.toUpgrade[id] = struct{}{} + + return nil +} + +func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreCommitInfo) big.Int { + replace := m.maybeUpgradableSector() + if replace != nil { + loc, err := m.api.StateSectorPartition(ctx, m.maddr, *replace, nil) + if err != nil { + log.Errorf("error calling StateSectorPartition for replaced sector: %+v", err) + return big.Zero() + } + + params.ReplaceCapacity = true + params.ReplaceSectorNumber = *replace + params.ReplaceSectorDeadline = loc.Deadline + params.ReplaceSectorPartition = loc.Partition + + ri, err := m.GetSectorInfo(*replace) + if err != nil { + log.Errorf("error calling GetSectorInfo for replaced sector: %+v", err) + return big.Zero() + } + + if params.Expiration < ri.PreCommitInfo.Expiration { + // TODO: Some limit on this + params.Expiration = ri.PreCommitInfo.Expiration + } + + return ri.PreCommitDeposit + } + + return big.Zero() +} + +func (m *Sealing) maybeUpgradableSector() *abi.SectorNumber { + m.upgradeLk.Lock() + defer m.upgradeLk.Unlock() + for number := range m.toUpgrade { + // TODO: checks to match actor constraints + + // this one looks good + /*if checks */ + { + delete(m.toUpgrade, number) + return &number + } + } + + return nil +} diff --git a/extern/storage-fsm/utils.go b/extern/storage-fsm/utils.go new file mode 100644 index 000000000..b507907fb --- /dev/null +++ b/extern/storage-fsm/utils.go @@ -0,0 +1,57 @@ +package sealing + +import ( + "math/bits" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) { + // Convert to in-sector bytes for easier math: + // + // Sector size to user bytes ratio is constant, e.g. for 1024B we have 1016B + // of user-usable data. + // + // (1024/1016 = 128/127) + // + // Given that we can get sector size by simply adding 1/127 of the user + // bytes + // + // (we convert to sector bytes as they are nice round binary numbers) + + toFill := uint64(in + (in / 127)) + + // We need to fill the sector with pieces that are powers of 2. Conveniently + // computers store numbers in binary, which means we can look at 1s to get + // all the piece sizes we need to fill the sector. It also means that number + // of pieces is the number of 1s in the number of remaining bytes to fill + out := make([]abi.UnpaddedPieceSize, bits.OnesCount64(toFill)) + for i := range out { + // Extract the next lowest non-zero bit + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << next + // e.g: if the number is 0b010100, psize will be 0b000100 + + // set that bit to 0 by XORing it, so the next iteration looks at the + // next bit + toFill ^= psize + + // Add the piece size to the list of pieces we need to create + out[i] = abi.PaddedPieceSize(psize).Unpadded() + } + return out, nil +} + +func (m *Sealing) ListSectors() ([]SectorInfo, error) { + var sectors []SectorInfo + if err := m.sectors.List(§ors); err != nil { + return nil, err + } + return sectors, nil +} + +func (m *Sealing) GetSectorInfo(sid abi.SectorNumber) (SectorInfo, error) { + var out SectorInfo + err := m.sectors.Get(uint64(sid)).Get(&out) + return out, err +} diff --git a/extern/storage-fsm/utils_test.go b/extern/storage-fsm/utils_test.go new file mode 100644 index 000000000..1d6b6c515 --- /dev/null +++ b/extern/storage-fsm/utils_test.go @@ -0,0 +1,45 @@ +package sealing + +import ( + "testing" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/stretchr/testify/assert" +) + +func testFill(t *testing.T, n abi.UnpaddedPieceSize, exp []abi.UnpaddedPieceSize) { + f, err := fillersFromRem(n) + assert.NoError(t, err) + assert.Equal(t, exp, f) + + var sum abi.UnpaddedPieceSize + for _, u := range f { + sum += u + } + assert.Equal(t, n, sum) +} + +func TestFillersFromRem(t *testing.T) { + for i := 8; i < 32; i++ { + // single + ub := abi.PaddedPieceSize(uint64(1) << i).Unpadded() + testFill(t, ub, []abi.UnpaddedPieceSize{ub}) + + // 2 + ub = abi.PaddedPieceSize(uint64(5) << i).Unpadded() + ub1 := abi.PaddedPieceSize(uint64(1) << i).Unpadded() + ub3 := abi.PaddedPieceSize(uint64(4) << i).Unpadded() + testFill(t, ub, []abi.UnpaddedPieceSize{ub1, ub3}) + + // 4 + ub = abi.PaddedPieceSize(uint64(15) << i).Unpadded() + ub2 := abi.PaddedPieceSize(uint64(2) << i).Unpadded() + ub4 := abi.PaddedPieceSize(uint64(8) << i).Unpadded() + testFill(t, ub, []abi.UnpaddedPieceSize{ub1, ub2, ub3, ub4}) + + // different 2 + ub = abi.PaddedPieceSize(uint64(9) << i).Unpadded() + testFill(t, ub, []abi.UnpaddedPieceSize{ub1, ub4}) + } +} diff --git a/go.mod b/go.mod index 199f69140..51ca85830 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( github.com/libp2p/go-libp2p-mplex v0.2.4 github.com/libp2p/go-libp2p-peer v0.2.0 github.com/libp2p/go-libp2p-peerstore v0.2.6 - github.com/libp2p/go-libp2p-pubsub v0.3.4-0.20200731161531-2b5243c72f0d + github.com/libp2p/go-libp2p-pubsub v0.3.4 github.com/libp2p/go-libp2p-quic-transport v0.7.1 github.com/libp2p/go-libp2p-record v0.1.3 github.com/libp2p/go-libp2p-routing-helpers v0.2.3 @@ -136,4 +136,8 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi +replace github.com/filecoin-project/sector-storage => ./extern/sector-storage + +replace github.com/filecoin-project/storage-fsm => ./extern/storage-fsm + replace github.com/dgraph-io/badger/v2 => github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200716180832-3ab515320794 diff --git a/go.sum b/go.sum index 46f8be5e4..ca9f0df33 100644 --- a/go.sum +++ b/go.sum @@ -229,7 +229,6 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20200731171407-e559a0579161/ github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.3/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= -github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.1.2 h1:TjLregCoyP1/5lm7WCM0axyV1myIHwbjGa21skuu5tk= github.com/filecoin-project/go-bitfield v0.1.2/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= @@ -249,7 +248,6 @@ github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0 github.com/filecoin-project/go-multistore v0.0.3/go.mod h1:kaNqCC4IhU4B1uyr7YWFHd23TL4KM32aChS0jNkyUvQ= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= -github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261 h1:A256QonvzRaknIIAuWhe/M2dpV2otzs3NBhi5TWa/UA= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= @@ -261,10 +259,6 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg= github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8= -github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo= -github.com/filecoin-project/sector-storage v0.0.0-20200730050024-3ee28c3b6d9a/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA= -github.com/filecoin-project/sector-storage v0.0.0-20200805173933-deec7a2658d4 h1:vEqr7sPz9sFEe8unfvY5X4DmkQrhmfLvbsXlmVMtmqM= -github.com/filecoin-project/sector-storage v0.0.0-20200805173933-deec7a2658d4/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA= github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y= github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY= @@ -274,8 +268,6 @@ github.com/filecoin-project/specs-actors v0.8.7-0.20200805174427-9d42fb163883 h1 github.com/filecoin-project/specs-actors v0.8.7-0.20200805174427-9d42fb163883/go.mod h1:QRihI/fadrhWzt7HH6mT32upOdDFpSYCFnr3JEI1L50= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY= github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k= -github.com/filecoin-project/storage-fsm v0.0.0-20200805013058-9d9ea4e6331f h1:WeMFRLMtAFqUwobouSeYj3pfgYtsSUwi3ztqDzFJMZY= -github.com/filecoin-project/storage-fsm v0.0.0-20200805013058-9d9ea4e6331f/go.mod h1:1CGbd11KkHuyWPT+xwwCol1zl/jnlpiKD2L4fzKxaiI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= @@ -867,8 +859,8 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato= -github.com/libp2p/go-libp2p-pubsub v0.3.4-0.20200731161531-2b5243c72f0d h1:1kfMc74C1DZGh97VJpA5efPXWU3tmdRF/wKYbFYya/4= -github.com/libp2p/go-libp2p-pubsub v0.3.4-0.20200731161531-2b5243c72f0d/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= +github.com/libp2p/go-libp2p-pubsub v0.3.4 h1:8PollxXtUvzy0DMn5XFMg/JihjaKboWyk3ML6yRW1Lk= +github.com/libp2p/go-libp2p-pubsub v0.3.4/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI= github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M= github.com/libp2p/go-libp2p-quic-transport v0.7.1 h1:X6Ond9GANspXpgwJlSR9yxcMMD6SLBnGKRtwjBG5awc= @@ -1481,7 +1473,6 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1632,7 +1623,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1694,7 +1684,6 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/node/impl/full/gas.go b/node/impl/full/gas.go index c14c147a0..49f07b271 100644 --- a/node/impl/full/gas.go +++ b/node/impl/full/gas.go @@ -59,7 +59,7 @@ func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxq return out, nil } -func (a *GasAPI) GasEsitmateGasPremium(ctx context.Context, nblocksincl uint64, +func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, _ types.TipSetKey) (types.BigInt, error) { if nblocksincl == 0 { diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index ac1bb58eb..caf4255f3 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -2,6 +2,7 @@ package full import ( "context" + "sync" "github.com/ipfs/go-cid" "go.uber.org/fx" @@ -23,6 +24,11 @@ type MpoolAPI struct { Chain *store.ChainStore Mpool *messagepool.MessagePool + + PushLocks struct { + m map[address.Address]chan struct{} + sync.Mutex + } `name:"verymuchunique" optional:"true"` } func (a *MpoolAPI) MpoolGetConfig(context.Context) (*types.MpoolConfig, error) { @@ -105,10 +111,35 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci return a.Mpool.Push(smsg) } -// GasMargin sets by how much should gas used be increased over test execution -var GasMargin = 1.5 - func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) { + { + fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil) + if err != nil { + return nil, xerrors.Errorf("getting key address: %w", err) + } + + a.PushLocks.Lock() + if a.PushLocks.m == nil { + a.PushLocks.m = make(map[address.Address]chan struct{}) + } + lk, ok := a.PushLocks.m[fromA] + if !ok { + lk = make(chan struct{}, 1) + a.PushLocks.m[msg.From] = lk + } + a.PushLocks.Unlock() + + select { + case lk <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + defer func() { + <-lk + }() + } + if msg.Nonce != 0 { return nil, xerrors.Errorf("MpoolPushMessage expects message nonce to be 0, was %d", msg.Nonce) } @@ -117,11 +148,11 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message) (*t if err != nil { return nil, xerrors.Errorf("estimating gas used: %w", err) } - msg.GasLimit = int64(float64(gasLimit) * GasMargin) + msg.GasLimit = int64(float64(gasLimit) * a.Mpool.GetConfig().GasLimitOverestimation) } if msg.GasPremium == types.EmptyInt || types.BigCmp(msg.GasPremium, types.NewInt(0)) == 0 { - gasPremium, err := a.GasEsitmateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) + gasPremium, err := a.GasEstimateGasPremium(ctx, 2, msg.From, msg.GasLimit, types.TipSetKey{}) if err != nil { return nil, xerrors.Errorf("estimating gas price: %w", err) } @@ -129,7 +160,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message) (*t } if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 { - feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK) + feeCap, err := a.GasEstimateFeeCap(ctx, msg, 10, types.EmptyTSK) if err != nil { return nil, xerrors.Errorf("estimating fee cap: %w", err) } diff --git a/node/impl/paych/paych.go b/node/impl/paych/paych.go index 8e28979f5..e7f39e400 100644 --- a/node/impl/paych/paych.go +++ b/node/impl/paych/paych.go @@ -116,25 +116,7 @@ func (a *PaychAPI) PaychSettle(ctx context.Context, addr address.Address) (cid.C } func (a *PaychAPI) PaychCollect(ctx context.Context, addr address.Address) (cid.Cid, error) { - - ci, err := a.PaychMgr.GetChannelInfo(addr) - if err != nil { - return cid.Undef, err - } - - msg := &types.Message{ - To: addr, - From: ci.Control, - Value: types.NewInt(0), - Method: builtin.MethodsPaych.Collect, - } - - smsg, err := a.MpoolPushMessage(ctx, msg) - if err != nil { - return cid.Undef, err - } - - return smsg.Cid(), nil + return a.PaychMgr.Collect(ctx, addr) } func (a *PaychAPI) PaychVoucherCheckValid(ctx context.Context, ch address.Address, sv *paych.SignedVoucher) error { diff --git a/paychmgr/manager.go b/paychmgr/manager.go index 7f47640f1..a75c79515 100644 --- a/paychmgr/manager.go +++ b/paychmgr/manager.go @@ -242,3 +242,11 @@ func (pm *Manager) Settle(ctx context.Context, addr address.Address) (cid.Cid, e } return ca.settle(ctx, addr) } + +func (pm *Manager) Collect(ctx context.Context, addr address.Address) (cid.Cid, error) { + ca, err := pm.accessorByAddress(addr) + if err != nil { + return cid.Undef, err + } + return ca.collect(ctx, addr) +} diff --git a/paychmgr/paych.go b/paychmgr/paych.go index f1d5199f6..ba2018e96 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -417,3 +417,27 @@ func (ca *channelAccessor) settle(ctx context.Context, ch address.Address) (cid. return smgs.Cid(), err } + +func (ca *channelAccessor) collect(ctx context.Context, ch address.Address) (cid.Cid, error) { + ca.lk.Lock() + defer ca.lk.Unlock() + + ci, err := ca.store.ByAddress(ch) + if err != nil { + return cid.Undef, err + } + + msg := &types.Message{ + To: ch, + From: ci.Control, + Value: types.NewInt(0), + Method: builtin.MethodsPaych.Collect, + } + + smsg, err := ca.api.MpoolPushMessage(ctx, msg) + if err != nil { + return cid.Undef, err + } + + return smsg.Cid(), nil +} diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 3dd061259..ab4c65c77 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -4,6 +4,7 @@ import ( "context" "sync" "testing" + "time" cborrpc "github.com/filecoin-project/go-cbor-util" @@ -741,3 +742,292 @@ func TestPaychGetWaitCtx(t *testing.T) { _, err = mgr.GetPaychWaitReady(ctx, mcid) require.Error(t, ctx.Err(), err) } + +// TestPaychGetMergeAddFunds tests that if a create channel is in +// progress and two add funds are queued up behind it, the two add funds +// will be merged +func TestPaychGetMergeAddFunds(t *testing.T) { + ctx := context.Background() + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewIDAddr(t, 101) + to := tutils.NewIDAddr(t, 102) + + sm := newMockStateManager() + pchapi := newMockPaychAPI() + defer pchapi.close() + + mgr, err := newManager(sm, store, pchapi) + require.NoError(t, err) + + // Send create message for a channel with value 10 + createAmt := big.NewInt(10) + _, createMsgCid, err := mgr.GetPaych(ctx, from, to, createAmt) + require.NoError(t, err) + + // Queue up two add funds requests behind create channel + //var addFundsQueuedUp sync.WaitGroup + //addFundsQueuedUp.Add(2) + var addFundsSent sync.WaitGroup + addFundsSent.Add(2) + + addFundsAmt1 := big.NewInt(5) + addFundsAmt2 := big.NewInt(3) + var addFundsCh1 address.Address + var addFundsCh2 address.Address + var addFundsMcid1 cid.Cid + var addFundsMcid2 cid.Cid + go func() { + //go addFundsQueuedUp.Done() + defer addFundsSent.Done() + + // Request add funds - should block until create channel has completed + addFundsCh1, addFundsMcid1, err = mgr.GetPaych(ctx, from, to, addFundsAmt1) + require.NoError(t, err) + }() + + go func() { + //go addFundsQueuedUp.Done() + defer addFundsSent.Done() + + // Request add funds again - should merge with waiting add funds request + addFundsCh2, addFundsMcid2, err = mgr.GetPaych(ctx, from, to, addFundsAmt2) + require.NoError(t, err) + }() + // Wait for add funds requests to be queued up + waitForQueueSize(t, mgr, from, to, 2) + + // Send create channel response + response := testChannelResponse(t, ch) + pchapi.receiveMsgResponse(createMsgCid, response) + + // Wait for create channel response + chres, err := mgr.GetPaychWaitReady(ctx, createMsgCid) + require.NoError(t, err) + require.Equal(t, ch, chres) + + // Wait for add funds requests to be sent + addFundsSent.Wait() + + // Expect add funds requests to have same channel as create channel and + // same message cid as each other (because they should have been merged) + require.Equal(t, ch, addFundsCh1) + require.Equal(t, ch, addFundsCh2) + require.Equal(t, addFundsMcid1, addFundsMcid2) + + // Send success add funds response + pchapi.receiveMsgResponse(addFundsMcid1, types.MessageReceipt{ + ExitCode: 0, + Return: []byte{}, + }) + + // Wait for add funds response + addFundsCh, err := mgr.GetPaychWaitReady(ctx, addFundsMcid1) + require.NoError(t, err) + require.Equal(t, ch, addFundsCh) + + // Make sure that one create channel message and one add funds message was + // sent + require.Equal(t, 2, pchapi.pushedMessageCount()) + + // Check create message amount is correct + createMsg := pchapi.pushedMessages(createMsgCid) + require.Equal(t, from, createMsg.Message.From) + require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, createAmt, createMsg.Message.Value) + + // Check merged add funds amount is the sum of the individual + // amounts + addFundsMsg := pchapi.pushedMessages(addFundsMcid1) + require.Equal(t, from, addFundsMsg.Message.From) + require.Equal(t, ch, addFundsMsg.Message.To) + require.Equal(t, types.BigAdd(addFundsAmt1, addFundsAmt2), addFundsMsg.Message.Value) +} + +// TestPaychGetMergeAddFundsCtxCancelOne tests that when a queued add funds +// request is cancelled, its amount is removed from the total merged add funds +func TestPaychGetMergeAddFundsCtxCancelOne(t *testing.T) { + ctx := context.Background() + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewIDAddr(t, 101) + to := tutils.NewIDAddr(t, 102) + + sm := newMockStateManager() + pchapi := newMockPaychAPI() + defer pchapi.close() + + mgr, err := newManager(sm, store, pchapi) + require.NoError(t, err) + + // Send create message for a channel with value 10 + createAmt := big.NewInt(10) + _, createMsgCid, err := mgr.GetPaych(ctx, from, to, createAmt) + require.NoError(t, err) + + // Queue up two add funds requests behind create channel + var addFundsSent sync.WaitGroup + addFundsSent.Add(2) + + addFundsAmt1 := big.NewInt(5) + addFundsAmt2 := big.NewInt(3) + var addFundsCh2 address.Address + var addFundsMcid2 cid.Cid + var addFundsErr1 error + addFundsCtx1, cancelAddFundsCtx1 := context.WithCancel(ctx) + go func() { + defer addFundsSent.Done() + + // Request add funds - should block until create channel has completed + _, _, addFundsErr1 = mgr.GetPaych(addFundsCtx1, from, to, addFundsAmt1) + }() + + go func() { + defer addFundsSent.Done() + + // Request add funds again - should merge with waiting add funds request + addFundsCh2, addFundsMcid2, err = mgr.GetPaych(ctx, from, to, addFundsAmt2) + require.NoError(t, err) + }() + // Wait for add funds requests to be queued up + waitForQueueSize(t, mgr, from, to, 2) + + // Cancel the first add funds request + cancelAddFundsCtx1() + + // Send create channel response + response := testChannelResponse(t, ch) + pchapi.receiveMsgResponse(createMsgCid, response) + + // Wait for create channel response + chres, err := mgr.GetPaychWaitReady(ctx, createMsgCid) + require.NoError(t, err) + require.Equal(t, ch, chres) + + // Wait for add funds requests to be sent + addFundsSent.Wait() + + // Expect first add funds request to have been cancelled + require.NotNil(t, addFundsErr1) + require.Equal(t, ch, addFundsCh2) + + // Send success add funds response + pchapi.receiveMsgResponse(addFundsMcid2, types.MessageReceipt{ + ExitCode: 0, + Return: []byte{}, + }) + + // Wait for add funds response + addFundsCh, err := mgr.GetPaychWaitReady(ctx, addFundsMcid2) + require.NoError(t, err) + require.Equal(t, ch, addFundsCh) + + // Make sure that one create channel message and one add funds message was + // sent + require.Equal(t, 2, pchapi.pushedMessageCount()) + + // Check create message amount is correct + createMsg := pchapi.pushedMessages(createMsgCid) + require.Equal(t, from, createMsg.Message.From) + require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, createAmt, createMsg.Message.Value) + + // Check merged add funds amount only includes the second add funds amount + // (because first was cancelled) + addFundsMsg := pchapi.pushedMessages(addFundsMcid2) + require.Equal(t, from, addFundsMsg.Message.From) + require.Equal(t, ch, addFundsMsg.Message.To) + require.Equal(t, addFundsAmt2, addFundsMsg.Message.Value) +} + +// TestPaychGetMergeAddFundsCtxCancelAll tests that when all queued add funds +// requests are cancelled, no add funds message is sent +func TestPaychGetMergeAddFundsCtxCancelAll(t *testing.T) { + ctx := context.Background() + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewIDAddr(t, 101) + to := tutils.NewIDAddr(t, 102) + + sm := newMockStateManager() + pchapi := newMockPaychAPI() + defer pchapi.close() + + mgr, err := newManager(sm, store, pchapi) + require.NoError(t, err) + + // Send create message for a channel with value 10 + createAmt := big.NewInt(10) + _, createMsgCid, err := mgr.GetPaych(ctx, from, to, createAmt) + require.NoError(t, err) + + // Queue up two add funds requests behind create channel + var addFundsSent sync.WaitGroup + addFundsSent.Add(2) + + var addFundsErr1 error + var addFundsErr2 error + addFundsCtx1, cancelAddFundsCtx1 := context.WithCancel(ctx) + addFundsCtx2, cancelAddFundsCtx2 := context.WithCancel(ctx) + go func() { + defer addFundsSent.Done() + + // Request add funds - should block until create channel has completed + _, _, addFundsErr1 = mgr.GetPaych(addFundsCtx1, from, to, big.NewInt(5)) + }() + + go func() { + defer addFundsSent.Done() + + // Request add funds again - should merge with waiting add funds request + _, _, addFundsErr2 = mgr.GetPaych(addFundsCtx2, from, to, big.NewInt(3)) + require.NoError(t, err) + }() + // Wait for add funds requests to be queued up + waitForQueueSize(t, mgr, from, to, 2) + + // Cancel all add funds requests + cancelAddFundsCtx1() + cancelAddFundsCtx2() + + // Send create channel response + response := testChannelResponse(t, ch) + pchapi.receiveMsgResponse(createMsgCid, response) + + // Wait for create channel response + chres, err := mgr.GetPaychWaitReady(ctx, createMsgCid) + require.NoError(t, err) + require.Equal(t, ch, chres) + + // Wait for add funds requests to error out + addFundsSent.Wait() + + require.NotNil(t, addFundsErr1) + require.NotNil(t, addFundsErr2) + + // Make sure that just the create channel message was sent + require.Equal(t, 1, pchapi.pushedMessageCount()) + + // Check create message amount is correct + createMsg := pchapi.pushedMessages(createMsgCid) + require.Equal(t, from, createMsg.Message.From) + require.Equal(t, builtin.InitActorAddr, createMsg.Message.To) + require.Equal(t, createAmt, createMsg.Message.Value) +} + +// waitForQueueSize waits for the funds request queue to be of the given size +func waitForQueueSize(t *testing.T, mgr *Manager, from address.Address, to address.Address, size int) { + ca, err := mgr.accessorByFromTo(from, to) + require.NoError(t, err) + + for { + if ca.queueSize() == size { + return + } + + time.Sleep(time.Millisecond) + } +} diff --git a/paychmgr/simple.go b/paychmgr/simple.go index 61c0a8fb1..12ff40d82 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "sync" "golang.org/x/sync/errgroup" @@ -36,15 +37,141 @@ type paychFundsRes struct { err error } -type onCompleteFn func(*paychFundsRes) - // fundsReq is a request to create a channel or add funds to a channel type fundsReq struct { - ctx context.Context - from address.Address - to address.Address - amt types.BigInt - onComplete onCompleteFn + ctx context.Context + promise chan *paychFundsRes + from address.Address + to address.Address + amt types.BigInt + + lk sync.Mutex + // merge parent, if this req is part of a merge + merge *mergedFundsReq + // whether the req's context has been cancelled + active bool +} + +func newFundsReq(ctx context.Context, from address.Address, to address.Address, amt types.BigInt) *fundsReq { + promise := make(chan *paychFundsRes) + return &fundsReq{ + ctx: ctx, + promise: promise, + from: from, + to: to, + amt: amt, + active: true, + } +} + +// onComplete is called when the funds request has been executed +func (r *fundsReq) onComplete(res *paychFundsRes) { + select { + case <-r.ctx.Done(): + case r.promise <- res: + } +} + +// cancel is called when the req's context is cancelled +func (r *fundsReq) cancel() { + r.lk.Lock() + + r.active = false + m := r.merge + + r.lk.Unlock() + + // If there's a merge parent, tell the merge parent to check if it has any + // active reqs left + if m != nil { + m.checkActive() + } +} + +// isActive indicates whether the req's context has been cancelled +func (r *fundsReq) isActive() bool { + r.lk.Lock() + defer r.lk.Unlock() + + return r.active +} + +// setMergeParent sets the merge that this req is part of +func (r *fundsReq) setMergeParent(m *mergedFundsReq) { + r.lk.Lock() + defer r.lk.Unlock() + + r.merge = m +} + +// mergedFundsReq merges together multiple add funds requests that are queued +// up, so that only one message is sent for all the requests (instead of one +// message for each request) +type mergedFundsReq struct { + ctx context.Context + cancel context.CancelFunc + reqs []*fundsReq +} + +func newMergedFundsReq(reqs []*fundsReq) *mergedFundsReq { + ctx, cancel := context.WithCancel(context.Background()) + m := &mergedFundsReq{ + ctx: ctx, + cancel: cancel, + reqs: reqs, + } + + for _, r := range m.reqs { + r.setMergeParent(m) + } + + // If the requests were all cancelled while being added, cancel the context + // immediately + m.checkActive() + + return m +} + +// Called when a fundsReq is cancelled +func (m *mergedFundsReq) checkActive() { + // Check if there are any active fundsReqs + for _, r := range m.reqs { + if r.isActive() { + return + } + } + + // If all fundsReqs have been cancelled, cancel the context + m.cancel() +} + +// onComplete is called when the queue has executed the mergeFundsReq. +// Calls onComplete on each fundsReq in the mergeFundsReq. +func (m *mergedFundsReq) onComplete(res *paychFundsRes) { + for _, r := range m.reqs { + if r.isActive() { + r.onComplete(res) + } + } +} + +func (m *mergedFundsReq) from() address.Address { + return m.reqs[0].from +} + +func (m *mergedFundsReq) to() address.Address { + return m.reqs[0].to +} + +// sum is the sum of the amounts in all requests in the merge +func (m *mergedFundsReq) sum() types.BigInt { + sum := types.NewInt(0) + for _, r := range m.reqs { + if r.isActive() { + sum = types.BigAdd(sum, r.amt) + } + } + return sum } // getPaych ensures that a channel exists between the from and to addresses, @@ -60,65 +187,97 @@ type fundsReq struct { // be attempted. func (ca *channelAccessor) getPaych(ctx context.Context, from, to address.Address, amt types.BigInt) (address.Address, cid.Cid, error) { // Add the request to add funds to a queue and wait for the result - promise := ca.enqueue(&fundsReq{ctx: ctx, from: from, to: to, amt: amt}) + freq := newFundsReq(ctx, from, to, amt) + ca.enqueue(freq) select { - case res := <-promise: + case res := <-freq.promise: return res.channel, res.mcid, res.err case <-ctx.Done(): + freq.cancel() return address.Undef, cid.Undef, ctx.Err() } } -// Queue up an add funds operation -func (ca *channelAccessor) enqueue(task *fundsReq) chan *paychFundsRes { - promise := make(chan *paychFundsRes) - task.onComplete = func(res *paychFundsRes) { - select { - case <-task.ctx.Done(): - case promise <- res: - } - } - +// Queue up an add funds operations +func (ca *channelAccessor) enqueue(task *fundsReq) { ca.lk.Lock() defer ca.lk.Unlock() ca.fundsReqQueue = append(ca.fundsReqQueue, task) - go ca.processNextQueueItem() - - return promise + go ca.processQueue() } -// Run the operation at the head of the queue -func (ca *channelAccessor) processNextQueueItem() { +// Run the operations in the queue +func (ca *channelAccessor) processQueue() { ca.lk.Lock() defer ca.lk.Unlock() + // Remove cancelled requests + ca.filterQueue() + + // If there's nothing in the queue, bail out if len(ca.fundsReqQueue) == 0 { return } - head := ca.fundsReqQueue[0] - res := ca.processTask(head.ctx, head.from, head.to, head.amt) + // Merge all pending requests into one. + // For example if there are pending requests for 3, 2, 4 then + // amt = 3 + 2 + 4 = 9 + merged := newMergedFundsReq(ca.fundsReqQueue[:]) + amt := merged.sum() + if amt.IsZero() { + // Note: The amount can be zero if requests are cancelled as we're + // building the mergedFundsReq + return + } + + res := ca.processTask(merged.ctx, merged.from(), merged.to(), amt) // If the task is waiting on an external event (eg something to appear on // chain) it will return nil if res == nil { // Stop processing the fundsReqQueue and wait. When the event occurs it will - // call processNextQueueItem() again + // call processQueue() again return } - // The task has finished processing so clean it up - ca.fundsReqQueue[0] = nil // allow GC of element - ca.fundsReqQueue = ca.fundsReqQueue[1:] + // Finished processing so clear the queue + ca.fundsReqQueue = nil // Call the task callback with its results - head.onComplete(res) + merged.onComplete(res) +} - // Process the next task - if len(ca.fundsReqQueue) > 0 { - go ca.processNextQueueItem() +// filterQueue filters cancelled requests out of the queue +func (ca *channelAccessor) filterQueue() { + if len(ca.fundsReqQueue) == 0 { + return } + + // Remove cancelled requests + i := 0 + for _, r := range ca.fundsReqQueue { + if r.isActive() { + ca.fundsReqQueue[i] = r + i++ + } + } + + // Allow GC of remaining slice elements + for rem := i; rem < len(ca.fundsReqQueue); rem++ { + ca.fundsReqQueue[i] = nil + } + + // Resize slice + ca.fundsReqQueue = ca.fundsReqQueue[:i] +} + +// queueSize is the size of the funds request queue (used by tests) +func (ca *channelAccessor) queueSize() int { + ca.lk.Lock() + defer ca.lk.Unlock() + + return len(ca.fundsReqQueue) } // msgWaitComplete is called when the message for a previous task is confirmed @@ -139,7 +298,7 @@ func (ca *channelAccessor) msgWaitComplete(mcid cid.Cid, err error) { // The queue may have been waiting for msg completion to proceed, so // process the next queue item if len(ca.fundsReqQueue) > 0 { - go ca.processNextQueueItem() + go ca.processQueue() } } diff --git a/paychmgr/store.go b/paychmgr/store.go index d7c6e82e7..62c4cf9b2 100644 --- a/paychmgr/store.go +++ b/paychmgr/store.go @@ -328,8 +328,7 @@ func (ps *Store) ByChannelID(channelID string) (*ChannelInfo, error) { return unmarshallChannelInfo(&stored, res) } -// CreateChannel creates an outbound channel for the given from / to, ensuring -// it has a higher sequence number than any existing channel with the same from / to +// CreateChannel creates an outbound channel for the given from / to func (ps *Store) CreateChannel(from address.Address, to address.Address, createMsgCid cid.Cid, amt types.BigInt) (*ChannelInfo, error) { ci := &ChannelInfo{ Direction: DirOutbound, diff --git a/scripts/dev/api.bash b/scripts/dev/api.bash new file mode 100644 index 000000000..5539e4fef --- /dev/null +++ b/scripts/dev/api.bash @@ -0,0 +1,11 @@ +#!/bin/bash +# vim: set expandtab ts=2 sw=2: + +token=$(lotus auth create-token --perm admin) + +runAPI() { + curl -X POST \ + -H "Content-Type: application/json" \ + --data '{"jsonrpc":"2.0","id":2,"method":"Filecoin.'"$1"'","params":'"${2:-null}"'}' \ + 'http://127.0.0.1:1234/rpc/v0?token='"$token" +}