Merge pull request #1185 from filecoin-project/fork/snowball

WIP, fork, fix slashing power calc

build/forks.go

@@ -5,3 +5,5 @@ const ForkBlizzardHeight = 6288
 const ForkFrigidHeight = 7950
 
 const ForkBootyBayHeight = 11000
+
+const ForkMissingSnowballs = 34000

build/parameters.go

@@ -2,4 +2,6 @@ package build
 
 import rice "github.com/GeertJohan/go.rice"
 
-var ParametersJson = rice.MustFindBox("proof-params").MustBytes("parameters.json")
+func ParametersJson() []byte {
+	return rice.MustFindBox("proof-params").MustBytes("parameters.json")
+}

build/params_debug.go

@@ -2,6 +2,10 @@
 
 package build
 
+func init() {
+	InsecurePoStValidation = true
+}
+
 var SectorSizes = []uint64{1024}
 
 // Seconds

build/version.go

@@ -5,7 +5,7 @@ import "fmt"
 var CurrentCommit string
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.2.6"
+const BuildVersion = "0.2.7"
 
 var UserVersion = BuildVersion + CurrentCommit
 

chain/actors/actor_miner2.go

@@ -33,7 +33,10 @@ func (sma StorageMinerActor2) Exports() []interface{} {
 		//8:  sma.DePledge,
 		9:  sma.GetOwner,
 		10: sma.GetWorkerAddr,
-		11: sma.GetPower, // TODO: Remove
+		11: withUpdates(
+			update{0, sma.GetPower},
+			update{build.ForkMissingSnowballs, sma.GetPower2},
+		), // FORK
 		12: sma.GetPeerID,
 		13: sma.GetSectorSize,
 		14: sma.UpdatePeerID,
@@ -401,6 +404,20 @@ func (sma StorageMinerActor2) GetPower(act *types.Actor, vmctx types.VMContext,
 	if err != nil {
 		return nil, err
 	}
+
+	return self.Power.Bytes(), nil
+}
+
+func (sma StorageMinerActor2) GetPower2(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if self.SlashedAt != 0 {
+		return types.NewInt(0).Bytes(), nil
+	}
+
 	return self.Power.Bytes(), nil
 }
 
@@ -615,6 +632,11 @@ func (sma StorageMinerActor2) CheckMiner(act *types.Actor, vmctx types.VMContext
 	// Slash for being late
 
 	self.SlashedAt = vmctx.BlockHeight()
+	oldPower := self.Power
+
+	if vmctx.BlockHeight() > build.ForkMissingSnowballs {
+		self.Power = types.NewInt(0)
+	}
 
 	nstate, err := vmctx.Storage().Put(self)
 	if err != nil {
@@ -625,7 +647,7 @@ func (sma StorageMinerActor2) CheckMiner(act *types.Actor, vmctx types.VMContext
 	}
 
 	var out bytes.Buffer
-	if err := self.Power.MarshalCBOR(&out); err != nil {
+	if err := oldPower.MarshalCBOR(&out); err != nil {
 		return nil, aerrors.HandleExternalError(err, "marshaling return value")
 	}
 	return out.Bytes(), nil
@@ -710,6 +732,23 @@ func (sma StorageMinerActor2) SlashConsensusFault(act *types.Actor, vmctx types.
 		return nil, aerrors.Wrap(err, "failed to burn funds")
 	}
 
+	if vmctx.BlockHeight() > build.ForkMissingSnowballs {
+		oldstate, self, err := loadState(vmctx)
+		if err != nil {
+			return nil, aerrors.Wrap(err, "failed to load state for slashing")
+		}
+
+		self.Power = types.NewInt(0)
+
+		ncid, err := vmctx.Storage().Put(self)
+		if err != nil {
+			return nil, err
+		}
+		if err := vmctx.Storage().Commit(oldstate, ncid); err != nil {
+			return nil, err
+		}
+	}
+
 	// TODO: this still allows the miner to commit sectors and submit posts,
 	// their users could potentially be unaffected, but the miner will never be
 	// able to mine a block again

chain/actors/actor_storagepower.go

@@ -648,6 +648,8 @@ func checkProofSubmissionsAtH(vmctx types.VMContext, self *StoragePowerState, he
 		return aerrors.HandleExternalError(err, "failed to load proving bucket")
 	}
 
+	forRemoval := make([]address.Address, 0)
+
 	err = bhamt.ForEach(vmctx.Context(), func(k string, val interface{}) error {
 		_, span := trace.StartSpan(vmctx.Context(), "StoragePowerActor.CheckProofSubmissions.loop")
 		defer span.End()
@@ -657,6 +659,18 @@ func checkProofSubmissionsAtH(vmctx types.VMContext, self *StoragePowerState, he
 			return aerrors.Escalate(err, "parsing miner address")
 		}
 
+		if vmctx.BlockHeight() > build.ForkMissingSnowballs {
+			has, aerr := MinerSetHas(vmctx, self.Miners, maddr)
+			if aerr != nil {
+				return aerr
+			}
+
+			if !has {
+				forRemoval = append(forRemoval, maddr)
+			}
+
+		}
+
 		span.AddAttributes(trace.StringAttribute("miner", maddr.String()))
 
 		params, err := SerializeParams(&CheckMinerParams{NetworkPower: self.TotalStorage})
@@ -690,6 +704,24 @@ func checkProofSubmissionsAtH(vmctx types.VMContext, self *StoragePowerState, he
 		return aerrors.HandleExternalError(err, "iterating miners in proving bucket")
 	}
 
+	if vmctx.BlockHeight() > build.ForkMissingSnowballs && len(forRemoval) > 0 {
+		nBucket, err := MinerSetRemove(vmctx.Context(), vmctx, bucket, forRemoval...)
+
+		if err != nil {
+			return aerrors.Wrap(err, "could not remove miners from set")
+		}
+
+		eerr := buckets.Set(bucketID, nBucket)
+		if err != nil {
+			return aerrors.HandleExternalError(eerr, "could not set the bucket")
+		}
+		ncid, eerr := buckets.Flush()
+		if err != nil {
+			return aerrors.HandleExternalError(eerr, "could not flush buckets")
+		}
+		self.ProvingBuckets = ncid
+	}
+
 	return nil
 }
 
@@ -764,19 +796,21 @@ func MinerSetAdd(ctx context.Context, vmctx types.VMContext, rcid cid.Cid, maddr
 	return c, nil
 }
 
-func MinerSetRemove(ctx context.Context, vmctx types.VMContext, rcid cid.Cid, maddr address.Address) (cid.Cid, aerrors.ActorError) {
+func MinerSetRemove(ctx context.Context, vmctx types.VMContext, rcid cid.Cid, maddrs ...address.Address) (cid.Cid, aerrors.ActorError) {
 	nd, err := hamt.LoadNode(ctx, vmctx.Ipld(), rcid)
 	if err != nil {
 		return cid.Undef, aerrors.HandleExternalError(err, "failed to load miner set")
 	}
 
-	mkey := string(maddr.Bytes())
-	switch nd.Delete(ctx, mkey) {
-	default:
-		return cid.Undef, aerrors.HandleExternalError(err, "failed to delete miner from set")
-	case hamt.ErrNotFound:
-		return cid.Undef, aerrors.New(1, "miner not found in set on delete")
-	case nil:
+	for _, maddr := range maddrs {
+		mkey := string(maddr.Bytes())
+		switch nd.Delete(ctx, mkey) {
+		default:
+			return cid.Undef, aerrors.HandleExternalError(err, "failed to delete miner from set")
+		case hamt.ErrNotFound:
+			return cid.Undef, aerrors.New(1, "miner not found in set on delete")
+		case nil:
+		}
 	}
 
 	if err := nd.Flush(ctx); err != nil {

chain/actors/forks.go

@@ -26,7 +26,7 @@ func withUpdates(updates ...update) interface{} {
 		vmctx := args[1].Interface().(types.VMContext)
 
 		for _, u := range updates {
-			if vmctx.BlockHeight() >= u.start {
+			if vmctx.BlockHeight() > u.start {
 				return reflect.ValueOf(u.method).Call(args)
 			}
 		}

chain/stmgr/forks.go

@@ -7,6 +7,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
@@ -35,12 +36,69 @@ func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, he
 			if err != nil {
 				return cid.Undef, xerrors.Errorf("booty bay bug fix failed: %w", err)
 			}
+		case build.ForkMissingSnowballs:
+			log.Warnw("Adding more snow to the world", "height", i)
+			pstate, err = fixTooFewSnowballs(ctx, sm, pstate)
+			if err != nil {
+				return cid.Undef, xerrors.Errorf("missing snowballs bug fix failed: %w", err)
+			}
 		}
 	}
 
 	return pstate, nil
 }
 
+func fixTooFewSnowballs(ctx context.Context, sm *StateManager, pstate cid.Cid) (cid.Cid, error) {
+	cst := hamt.CSTFromBstore(sm.cs.Blockstore())
+	st, err := state.LoadStateTree(cst, pstate)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	spa, err := st.GetActor(actors.StoragePowerAddress)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get storage power actor: %w", err)
+	}
+
+	var spast actors.StoragePowerState
+	if err := cst.Get(ctx, spa.Head, &spast); err != nil {
+		return cid.Undef, err
+	}
+
+	miners, err := actors.MinerSetList(ctx, cst, spast.Miners)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	sum := types.NewInt(0)
+	for _, m := range miners {
+		mact, err := st.GetActor(m)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("getting miner actor to fix: %w", err)
+		}
+
+		var mstate actors.StorageMinerActorState
+		if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
+			return cid.Undef, xerrors.Errorf("failed to load miner actor state: %w", err)
+		}
+
+		if mstate.SlashedAt != 0 {
+			continue
+		}
+		sum = types.BigAdd(sum, mstate.Power)
+	}
+
+	spast.TotalStorage = sum
+	nspahead, err := cst.Put(ctx, &spast)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	spa.Head = nspahead
+
+	return st.Flush(ctx)
+}
+
 /*
 1) Iterate through each miner in the chain:
 	1.1) Fixup their sector set and proving set

cli/chain.go

@@ -388,6 +388,14 @@ func printTipSet(format string, ts *types.TipSet) {
 		blks += fmt.Sprintf("%s: %s,", b.Cid(), b.Miner)
 	}
 	blks += " ]"
+
+	sCids := make([]string, 0, len(blks))
+
+	for _, c := range ts.Cids() {
+		sCids = append(sCids, c.String())
+	}
+
+	format = strings.ReplaceAll(format, "<tipset>", strings.Join(sCids, ","))
 	format = strings.ReplaceAll(format, "<blocks>", blks)
 	format = strings.ReplaceAll(format, "<weight>", fmt.Sprint(ts.Blocks()[0].ParentWeight))
 

cli/params.go

@@ -24,7 +24,7 @@ var fetchParamCmd = &cli.Command{
 			return err
 		}
 		sectorSize := uint64(sectorSizeInt)
-		err = paramfetch.GetParams(build.ParametersJson, sectorSize)
+		err = paramfetch.GetParams(build.ParametersJson(), sectorSize)
 		if err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}

cli/state.go

@@ -46,6 +46,7 @@ var stateCmd = &cli.Command{
 		stateListMessagesCmd,
 		stateComputeStateCmd,
 		stateCallCmd,
+		stateGetDealSetCmd,
 	},
 }
 
@@ -299,6 +300,47 @@ var statePledgeCollateralCmd = &cli.Command{
 	},
 }
 
+var stateGetDealSetCmd = &cli.Command{
+	Name:  "get-deal",
+	Usage: "View on-chain deal info",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify miner to list sectors for")
+		}
+
+		dealid, err := strconv.ParseUint(cctx.Args().First(), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("parsing deal ID: %w", err)
+		}
+
+		ts, err := loadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		deal, err := api.StateMarketStorageDeal(ctx, dealid, ts)
+		if err != nil {
+			return err
+		}
+
+		data, err := json.MarshalIndent(deal, "", "  ")
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(data))
+
+		return nil
+	},
+}
+
 var stateListMinersCmd = &cli.Command{
 	Name:  "list-miners",
 	Usage: "list all miners in the network",

cmd/lotus-bench/main.go

@@ -155,7 +155,7 @@ func main() {
 				}
 			}
 
-			if err := paramfetch.GetParams(build.ParametersJson, sectorSize); err != nil {
+			if err := paramfetch.GetParams(build.ParametersJson(), sectorSize); err != nil {
 				return xerrors.Errorf("getting params: %w", err)
 			}
 			sb, err := sectorbuilder.New(cfg, mds)

cmd/lotus-chainwatch/storage.go

@@ -302,6 +302,37 @@ create index if not exists deal_activations_activation_epoch_index
 
 create unique index if not exists deal_activations_deal_uindex
 	on deal_activations (deal);
+
+create table if not exists blocks_challenges
+(
+	block text not null
+		constraint blocks_challenges_pk_2
+			primary key
+		constraint blocks_challenges_blocks_cid_fk
+			references blocks,
+	index bigint not null,
+	sector_id bigint not null,
+	partial bytea not null,
+	candidate bigint not null,
+	constraint blocks_challenges_pk
+		unique (block, index)
+);
+
+create index if not exists blocks_challenges_block_index
+	on blocks_challenges (block);
+
+create index if not exists blocks_challenges_block_candidate_index
+	on blocks_challenges (block,candidate);
+
+create index if not exists blocks_challenges_block_index_index
+	on blocks_challenges (block, index);
+
+create index if not exists blocks_challenges_candidate_index
+	on blocks_challenges (candidate);
+
+create index if not exists blocks_challenges_index_index
+	on blocks_challenges (index);
+
 `)
 	if err != nil {
 		return err
@@ -477,6 +508,7 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
 create temp table tbp (like block_parents excluding constraints) on commit drop;
 create temp table bs (like blocks_synced excluding constraints) on commit drop;
 create temp table b (like blocks excluding constraints) on commit drop;
+create temp table c (like blocks_challenges excluding constraints) on commit drop;
 
 
 `); err != nil {
@@ -527,7 +559,7 @@ create temp table b (like blocks excluding constraints) on commit drop;
 		}
 	}
 
-	stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) from stdin `)
+	stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) from stdin`)
 	if err != nil {
 		return err
 	}
@@ -564,10 +596,36 @@ create temp table b (like blocks excluding constraints) on commit drop;
 		return xerrors.Errorf("blk put: %w", err)
 	}
 
+	stmt3, err := tx.Prepare(`copy c (block, index, sector_id, partial, candidate) from stdin`)
+	if err != nil {
+		return xerrors.Errorf("s3 create: %w", err)
+	}
+	for _, bh := range bhs {
+		for index, c := range bh.EPostProof.Candidates {
+			if _, err := stmt3.Exec(
+				bh.Cid().String(),
+				index,
+				c.SectorID,
+				c.Partial,
+				c.ChallengeIndex); err != nil {
+				log.Error(err)
+			}
+		}
+	}
+
+	if err := stmt3.Close(); err != nil {
+		return xerrors.Errorf("s2 close: %w", err)
+	}
+
+	if _, err := tx.Exec(`insert into blocks_challenges select * from c on conflict do nothing `); err != nil {
+		return xerrors.Errorf("blk put: %w", err)
+	}
+
 	err = tx.Commit()
 	if err != nil {
 		return xerrors.Errorf("commit: %w", err)
 	}
+
 	return nil
 }
 

cmd/lotus-seal-worker/sub.go

@@ -41,7 +41,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut
 		return err
 	}
 
-	if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil {
+	if err := paramfetch.GetParams(build.ParametersJson(), ssize); err != nil {
 		return xerrors.Errorf("get params: %w", err)
 	}
 

cmd/lotus-shed/bigint.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"gopkg.in/urfave/cli.v2"
+)
+
+var bigIntParseCmd = &cli.Command{
+	Name:        "bigint",
+	Description: "parse encoded big ints",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "enc",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		val := cctx.Args().Get(0)
+
+		var dec []byte
+		switch cctx.String("enc") {
+		case "base64":
+			d, err := base64.StdEncoding.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding base64 value: %w", err)
+			}
+			dec = d
+		case "hex":
+			d, err := hex.DecodeString(val)
+			if err != nil {
+				return fmt.Errorf("decoding hex value: %w", err)
+			}
+			dec = d
+		default:
+			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		}
+
+		iv := types.BigFromBytes(dec)
+		fmt.Println(iv.String())
+		return nil
+	},
+}

cmd/lotus-shed/main.go

@@ -20,6 +20,7 @@ func main() {
 		keyinfoCmd,
 		peerkeyCmd,
 		noncefix,
+		bigIntParseCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-storage-miner/info.go

@@ -50,8 +50,8 @@ var infoCmd = &cli.Command{
 			return err
 		}
 
-		percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower)
-		fmt.Printf("Power: %s / %s (%0.4f%%)\n", pow.MinerPower.SizeStr(), pow.TotalPower.SizeStr(), float64(percI.Int64())/100000*10000)
+		percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000000)), pow.TotalPower)
+		fmt.Printf("Power: %s / %s (%0.4f%%)\n", pow.MinerPower.SizeStr(), pow.TotalPower.SizeStr(), float64(percI.Int64())/10000)
 
 		secCounts, err := api.StateMinerSectorCount(ctx, maddr, nil)
 		if err != nil {

cmd/lotus-storage-miner/init.go

@@ -97,7 +97,7 @@ var initCmd = &cli.Command{
 		}
 
 		log.Info("Checking proof parameters")
-		if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil {
+		if err := paramfetch.GetParams(build.ParametersJson(), ssize); err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}
 

cmd/lotus/daemon.go

@@ -90,6 +90,7 @@ var DaemonCmd = &cli.Command{
 		}
 
 		ctx := context.Background()
+		log.Infof("lotus repo: %s", cctx.String("repo"))
 		r, err := repo.NewFS(cctx.String("repo"))
 		if err != nil {
 			return xerrors.Errorf("opening fs repo: %w", err)
@@ -99,7 +100,7 @@ var DaemonCmd = &cli.Command{
 			return xerrors.Errorf("repo init error: %w", err)
 		}
 
-		if err := paramfetch.GetParams(build.ParametersJson, 0); err != nil {
+		if err := paramfetch.GetParams(build.ParametersJson(), 0); err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}
 

cmd/lotus/debug_advance.go

@@ -0,0 +1,112 @@
+// +build debug
+
+package main
+
+import (
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/gen"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/miner"
+	"golang.org/x/xerrors"
+
+	"gopkg.in/urfave/cli.v2"
+)
+
+func init() {
+	AdvanceBlockCmd = &cli.Command{
+		Name: "advance-block",
+		Action: func(cctx *cli.Context) error {
+			api, closer, err := lcli.GetFullNodeAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			ctx := lcli.ReqContext(cctx)
+			head, err := api.ChainHead(ctx)
+			if err != nil {
+				return err
+			}
+			pending, err := api.MpoolPending(ctx, head)
+			if err != nil {
+				return err
+			}
+
+			msgs, err := miner.SelectMessages(ctx, api.StateGetActor, head, pending)
+			if len(msgs) > build.BlockMessageLimit {
+				log.Error("SelectMessages returned too many messages: ", len(msgs))
+				msgs = msgs[:build.BlockMessageLimit]
+			}
+
+			addr, _ := address.NewIDAddress(101)
+			var ticket *types.Ticket
+			{
+				vrfBase := head.MinTicket().VRFProof
+				ret, err := api.StateCall(ctx, &types.Message{
+					From:   addr,
+					To:     addr,
+					Method: actors.MAMethods.GetWorkerAddr,
+				}, head)
+				if err != nil {
+					return xerrors.Errorf("failed to get miner worker addr: %w", err)
+				}
+
+				if ret.ExitCode != 0 {
+					return xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.ExitCode)
+				}
+
+				w, err := address.NewFromBytes(ret.Return)
+				if err != nil {
+					return xerrors.Errorf("GetWorkerAddr returned malformed address: %w", err)
+				}
+				t, err := gen.ComputeVRF(ctx, api.WalletSign, w, addr, gen.DSepTicket, vrfBase)
+				if err != nil {
+					return xerrors.Errorf("compute vrf failed: %w", err)
+				}
+				ticket = &types.Ticket{
+					VRFProof: t,
+				}
+
+			}
+
+			epostp := &types.EPostProof{
+				Proof: []byte("valid proof"),
+				Candidates: []types.EPostTicket{
+					{
+						ChallengeIndex: 0,
+						SectorID:       1,
+					},
+				},
+			}
+
+			{
+				r, err := api.ChainGetRandomness(ctx, head.Key(), int64(head.Height()+1)-build.EcRandomnessLookback)
+				if err != nil {
+					return xerrors.Errorf("chain get randomness: %w", err)
+				}
+				mworker, err := api.StateMinerWorker(ctx, addr, head)
+				if err != nil {
+					return xerrors.Errorf("failed to get miner worker: %w", err)
+				}
+
+				vrfout, err := gen.ComputeVRF(ctx, api.WalletSign, mworker, addr, gen.DSepElectionPost, r)
+				if err != nil {
+					return xerrors.Errorf("failed to compute VRF: %w", err)
+				}
+				epostp.PostRand = vrfout
+			}
+
+			uts := head.MinTimestamp() + uint64(build.BlockDelay)
+			nheight := head.Height() + 1
+			blk, err := api.MinerCreateBlock(ctx, addr, head, ticket, epostp, msgs, nheight, uts)
+			if err != nil {
+				return xerrors.Errorf("creating block: %w", err)
+			}
+
+			return api.SyncSubmitBlock(ctx, blk)
+		},
+	}
+}

cmd/lotus/main.go

@@ -14,12 +14,18 @@ import (
 	"github.com/filecoin-project/lotus/tracing"
 )
 
+var AdvanceBlockCmd *cli.Command
+
 func main() {
 	lotuslog.SetupLogLevels()
 
 	local := []*cli.Command{
 		DaemonCmd,
 	}
+	if AdvanceBlockCmd != nil {
+		local = append(local, AdvanceBlockCmd)
+	}
+
 	jaeger := tracing.SetupJaegerTracing("lotus")
 	defer func() {
 		if jaeger != nil {

documentation/en/.library.json

@@ -32,6 +32,12 @@
           "github": "en/install-lotus-ubuntu.md",
           "value": null
         },
+        {
+          "title": "Fedora Installation",
+          "slug": "en+install-lotus-fedora",
+          "github": "en/install-lotus-fedora.md",
+          "value": null
+        },
         {
           "title": "MacOS Installation",
           "slug": "en+install-lotus-macos",

documentation/en/storing-data.md

@@ -49,6 +49,12 @@ Store a **Data CID** with a miner:
 lotus client deal <Data CID> <miner> <price> <duration>
 ```
 
+Check the status of a deal:
+
+```sh
+lotus client list-deals
+```
+
 - Price is in attoFIL.
 - The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 45 seconds.
 

node/modules/storageminer.go

@@ -55,7 +55,7 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
 }
 
 func GetParams(sbc *sectorbuilder.Config) error {
-	if err := paramfetch.GetParams(build.ParametersJson, sbc.SectorSize); err != nil {
+	if err := paramfetch.GetParams(build.ParametersJson(), sbc.SectorSize); err != nil {
 		return xerrors.Errorf("fetching proof parameters: %w", err)
 	}
 

storage/sealing/fsm.go

@@ -205,6 +205,8 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
 			state.State = api.SealCommitFailed
 		case SectorSealFailed:
 			state.State = api.CommitFailed
+		case SectorCommitFailed:
+			state.State = api.CommitFailed
 		default:
 			return xerrors.Errorf("planCommitting got event of unknown type %T, events: %+v", event.User, events)
 		}

storage/sealing/fsm_test.go

@@ -83,3 +83,17 @@ func TestSeedRevert(t *testing.T) {
 	m.planSingle(SectorProving{})
 	require.Equal(m.t, m.state.State, api.Proving)
 }
+
+func TestPlanCommittingHandlesSectorCommitFailed(t *testing.T) {
+	m := test{
+		s:     &Sealing{},
+		t:     t,
+		state: &SectorInfo{State: api.Committing},
+	}
+
+	events := []statemachine.Event{{SectorCommitFailed{}}}
+
+	require.NoError(t, planCommitting(events, m.state))
+
+	require.Equal(t, api.SectorStates[api.CommitFailed], api.SectorStates[m.state.State])
+}

storage/sealing/garbage.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"math"
 	"math/rand"
+	"runtime"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"golang.org/x/xerrors"
@@ -14,6 +15,17 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+func (m *Sealing) pledgeReader(size uint64, parts uint64) io.Reader {
+	piece := sectorbuilder.UserBytesForSectorSize((size/127 + size) / parts)
+
+	readers := make([]io.Reader, parts)
+	for i := range readers {
+		readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece))
+	}
+
+	return io.MultiReader(readers...)
+}
+
 func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPieceSizes []uint64, sizes ...uint64) ([]Piece, error) {
 	if len(sizes) == 0 {
 		return nil, nil
@@ -21,10 +33,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
 
 	deals := make([]actors.StorageDealProposal, len(sizes))
 	for i, size := range sizes {
-		release := m.sb.RateLimit()
-		commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size)
-		release()
-
+		commP, err := m.fastPledgeCommitment(size, uint64(runtime.NumCPU()))
 		if err != nil {
 			return nil, err
 		}
@@ -81,7 +90,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID uint64, existingPie
 	out := make([]Piece, len(sizes))
 
 	for i, size := range sizes {
-		ppi, err := m.sb.AddPiece(size, sectorID, io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), existingPieceSizes)
+		ppi, err := m.sb.AddPiece(size, sectorID, m.pledgeReader(size, uint64(runtime.NumCPU())), existingPieceSizes)
 		if err != nil {
 			return nil, err
 		}

storage/sealing/utils.go

@@ -1,7 +1,12 @@
 package sealing
 
 import (
+	"io"
 	"math/bits"
+	"math/rand"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
@@ -42,6 +47,41 @@ func fillersFromRem(toFill uint64) ([]uint64, error) {
 	return out, nil
 }
 
+func (m *Sealing) fastPledgeCommitment(size uint64, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) {
+	parts = 1 << bits.Len64(parts) // round down to nearest power of 2
+
+	piece := sectorbuilder.UserBytesForSectorSize((size + size / 127) / parts)
+	out := make([]sectorbuilder.PublicPieceInfo, parts)
+	var lk sync.Mutex
+
+	var wg sync.WaitGroup
+	wg.Add(int(parts))
+	for i := uint64(0); i < parts; i++ {
+		go func(i uint64) {
+			defer wg.Done()
+
+			commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece)
+
+			lk.Lock()
+			if perr != nil {
+				err = multierror.Append(err, perr)
+			}
+			out[i] = sectorbuilder.PublicPieceInfo{
+				Size:  piece,
+				CommP: commP,
+			}
+			lk.Unlock()
+		}(i)
+	}
+	wg.Wait()
+
+	if err != nil {
+		return [32]byte{}, err
+	}
+
+	return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out)
+}
+
 func (m *Sealing) ListSectors() ([]SectorInfo, error) {
 	var sectors []SectorInfo
 	if err := m.sectors.List(&sectors); err != nil {

storage/sealing/utils_test.go

@@ -1,6 +1,7 @@
 package sealing
 
 import (
+	"github.com/filecoin-project/lotus/storage/sbmock"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -42,5 +43,13 @@ func TestFillersFromRem(t *testing.T) {
 		ub = sectorbuilder.UserBytesForSectorSize(uint64(9) << i)
 		testFill(t, ub, []uint64{ub1, ub4})
 	}
+}
 
+func TestFastPledge(t *testing.T) {
+	sz := uint64(16 << 20)
+
+	s := Sealing{sb: sbmock.NewMockSectorBuilder(0, sz)}
+	if _, err := s.fastPledgeCommitment(sectorbuilder.UserBytesForSectorSize(sz), 5); err != nil {
+		t.Fatalf("%+v", err)
+	}
 }

tools/dockers/README.md

@@ -0,0 +1,3 @@
+## About 
+
+In [docker-examples/](docker-examples/) are community-contributed Docker and Docker Compose examples.

tools/dockers/docker-examples/README.md

@@ -0,0 +1,34 @@
+## About 
+
+In this `docker-examples/` directory are community-contributed Docker and Docker Compose examples.
+
+#### Example Use Cases
+
+- **For a miner**
+   - basic storage miner (`miner-basic-`)
+   - distributed storage miner
+- **For API endpoints for building apps**
+   - local node for a developer (`api-local-`)
+   - hosted endpoint for apps / multiple developers (`api-hosted-`)
+- **For a local devnet or shared devnet**
+   - basic local devnet (also see [lotus docs on setting up a local devnet](https://lotu.sh/en+setup-local-dev-net))
+   - shared devnet
+  
+
+## Contributions
+
+- `miner-basic-busybox` - from @ldoublewood ([lotus PR 745](https://github.com/filecoin-project/lotus/pull/745))
+- `api-local-arch` - from @mateodelnorte
+- `api-hosted-debian-nginx` - from @RTradeLtd
+
+#### More examples:
+- [Issue 1143](https://github.com/filecoin-project/lotus/issues/1143)
+- [PR 1047](https://github.com/filecoin-project/lotus/pull/1047/files)
+
+## Contributing
+
+:whale: Have a new type of Docker to share?
+
+Make a PR with a new folder for it, describe its purpose and link to the repo or provide the Docker files. Add it to *Contributions* above.
+
+:beetle: See a bug? File an Issue so we can keep these up-to-date.

tools/dockers/docker-examples/api-hosted-debian-nginx/README.md

@@ -0,0 +1,10 @@
+## Description
+
+- **For API endpoints for building apps**
+  - hosted endpoint for multiple developers
+
+Another container spins up a simple block explorer.
+
+## Repo
+
+https://github.com/RTradeLtd/lotus-infra

tools/dockers/docker-examples/api-local-arch/README.md

@@ -0,0 +1,8 @@
+## Description
+
+- **For API endpoints for building apps**
+  - local node for a developer
+
+## Repo
+
+https://github.com/mateodelnorte/filecoin-lotus-docker

tools/dockers/docker-examples/basic-miner-busybox/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.4-buster
+FROM golang:1.13-buster
 MAINTAINER ldoublewood <ldoublewood@gmail.com>
 
 ENV SRC_DIR /lotus
@@ -44,10 +44,12 @@ RUN cd $SRC_DIR \
 
 COPY . $SRC_DIR
 
+ARG MAKE_TARGET=all
+
 # Build the thing.
 RUN cd $SRC_DIR \
   && . $HOME/.cargo/env \
-  && make
+  && make $MAKE_TARGET
 
 # Now comes the actual target image, which aims to be as small as possible.
 FROM busybox:1-glibc
@@ -56,7 +58,7 @@ MAINTAINER ldoublewood <ldoublewood@gmail.com>
 # Get the executable binary and TLS CAs from the build container.
 ENV SRC_DIR /lotus
 COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
-COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
+COPY --from=0 $SRC_DIR/lotus-* /usr/local/bin/
 COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
 COPY --from=0 /tmp/tini /sbin/tini
 COPY --from=0 /etc/ssl/certs /etc/ssl/certs
@@ -79,13 +81,15 @@ EXPOSE 5678
 ENV HOME_PATH /data
 ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
 
-RUN mkdir -p $HOME_PATH \
+RUN mkdir -p $HOME_PATH $PARAMCACHE_PATH \
   && adduser -D -h $HOME_PATH -u 1000 -G users lotus \
-  && chown lotus:users $HOME_PATH
+  && chown lotus:users $HOME_PATH $PARAMCACHE_PATH
 
 
 VOLUME $HOME_PATH
 VOLUME $PARAMCACHE_PATH
 
+USER lotus
+
 # Execute the daemon subcommand by default
 CMD ["/sbin/tini", "--", "lotus", "daemon"]

tools/dockers/docker-examples/basic-miner-busybox/README.md

@@ -0,0 +1,4 @@
+## Description
+
+- **For a miner**
+  - basic storage miner