Merge remote-tracking branch 'origin/next' into feat/dyn-base-fee

api/api_storage.go

@@ -73,6 +73,7 @@ type StorageMiner interface {
 	MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error
 	MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error)
 	MarketListRetrievalDeals(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)
+	MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error)
 	MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error)
 	MarketSetAsk(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error
 	MarketGetAsk(ctx context.Context) (*storagemarket.SignedStorageAsk, error)

api/apistruct/struct.go

@@ -218,6 +218,7 @@ type StorageMinerStruct struct {
 		MarketImportDealData      func(context.Context, cid.Cid, string) error                                                                                                                                 `perm:"write"`
 		MarketListDeals           func(ctx context.Context) ([]storagemarket.StorageDeal, error)                                                                                                               `perm:"read"`
 		MarketListRetrievalDeals  func(ctx context.Context) ([]retrievalmarket.ProviderDealState, error)                                                                                                       `perm:"read"`
+		MarketGetDealUpdates      func(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error)                                                                                                 `perm:"read"`
 		MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error)                                                                                                                 `perm:"read"`
 		MarketSetAsk              func(ctx context.Context, price types.BigInt, verifiedPrice types.BigInt, duration abi.ChainEpoch, minPieceSize abi.PaddedPieceSize, maxPieceSize abi.PaddedPieceSize) error `perm:"admin"`
 		MarketGetAsk              func(ctx context.Context) (*storagemarket.SignedStorageAsk, error)                                                                                                           `perm:"read"`
@@ -1005,6 +1006,10 @@ func (c *StorageMinerStruct) MarketListRetrievalDeals(ctx context.Context) ([]re
 	return c.Internal.MarketListRetrievalDeals(ctx)
 }
 
+func (c *StorageMinerStruct) MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error) {
+	return c.Internal.MarketGetDealUpdates(ctx, d)
+}
+
 func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
 	return c.Internal.MarketListIncompleteDeals(ctx)
 }

api/test/deals.go

@@ -144,6 +144,61 @@ func makeDeal(t *testing.T, ctx context.Context, rseed int, client *impl.FullNod
 	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, carExport, data)
 }
 
+func TestFastRetrievalDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
+	_ = os.Setenv("BELLMAN_NO_GPU", "1")
+
+	ctx := context.Background()
+	n, sn := b(t, 1, oneMiner)
+	client := n[0].FullNode.(*impl.FullNodeAPI)
+	miner := sn[0]
+
+	addrinfo, err := client.NetAddrsListen(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := miner.NetConnect(ctx, addrinfo); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Second)
+
+	mine := int64(1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for atomic.LoadInt64(&mine) == 1 {
+			time.Sleep(blocktime)
+			if err := sn[0].MineOne(ctx, MineNext); err != nil {
+				t.Error(err)
+			}
+		}
+	}()
+
+	data := make([]byte, 1600)
+	rand.New(rand.NewSource(int64(8))).Read(data)
+
+	r := bytes.NewReader(data)
+	fcid, err := client.ClientImportLocal(ctx, r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println("FILE CID: ", fcid)
+
+	deal := startDeal(t, ctx, miner, client, fcid, true)
+
+	waitDealPublished(t, ctx, miner, deal)
+	fmt.Println("deal published, retrieving")
+	// Retrieval
+	info, err := client.ClientGetDealInfo(ctx, *deal)
+	require.NoError(t, err)
+
+	testRetrieval(t, ctx, err, client, fcid, &info.PieceCID, false, data)
+	atomic.AddInt64(&mine, -1)
+	fmt.Println("shutting down mining")
+	<-done
+}
+
 func TestSenondDealRetrieval(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	_ = os.Setenv("BELLMAN_NO_GPU", "1")
 
@@ -275,6 +330,34 @@ loop:
 	}
 }
 
+func waitDealPublished(t *testing.T, ctx context.Context, miner TestStorageNode, deal *cid.Cid) {
+	subCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	updates, err := miner.MarketGetDealUpdates(subCtx, *deal)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for {
+		select {
+		case <-ctx.Done():
+			t.Fatal("context timeout")
+		case di := <-updates:
+			switch di.State {
+			case storagemarket.StorageDealProposalRejected:
+				t.Fatal("deal rejected")
+			case storagemarket.StorageDealFailing:
+				t.Fatal("deal failed")
+			case storagemarket.StorageDealError:
+				t.Fatal("deal errored", di.Message)
+			case storagemarket.StorageDealFinalizing, storagemarket.StorageDealSealing, storagemarket.StorageDealActive:
+				fmt.Println("COMPLETE", di)
+				return
+			}
+			fmt.Println("Deal state: ", storagemarket.DealStates[di.State])
+		}
+	}
+}
+
 func startSealingWaiting(t *testing.T, ctx context.Context, miner TestStorageNode) {
 	snums, err := miner.SectorsList(ctx)
 	require.NoError(t, err)

chain/sync.go

@@ -1300,7 +1300,7 @@ loop:
 	}
 
 	// base is the tipset in the candidate chain at the height equal to our known tipset height.
-	if base := blockSet[len(blockSet)-1]; !types.CidArrsEqual(base.Parents().Cids(), known.Cids()) {
+	if base := blockSet[len(blockSet)-1]; !types.CidArrsSubset(base.Parents().Cids(), known.Cids()) {
 		if base.Parents() == known.Parents() {
 			// common case: receiving a block thats potentially part of the same tipset as our best block
 			return blockSet, nil

chain/types/blockheader.go

@@ -178,6 +178,21 @@ func CidArrsEqual(a, b []cid.Cid) bool {
 	return true
 }
 
+func CidArrsSubset(a, b []cid.Cid) bool {
+	// order ignoring compare...
+	s := make(map[cid.Cid]bool)
+	for _, c := range b {
+		s[c] = true
+	}
+
+	for _, c := range a {
+		if !s[c] {
+			return false
+		}
+	}
+	return true
+}
+
 func CidArrsContains(a []cid.Cid, b cid.Cid) bool {
 	for _, elem := range a {
 		if elem.Equals(b) {

chain/vm/gas.go

@@ -84,46 +84,45 @@ type Pricelist interface {
 
 var prices = map[abi.ChainEpoch]Pricelist{
 	abi.ChainEpoch(0): &pricelistV0{
-		onChainMessageComputeBase:    137137,
+		onChainMessageComputeBase:    38863,
 		onChainMessageStorageBase:    36,
 		onChainMessageStoragePerByte: 1,
 
 		onChainReturnValuePerByte: 1,
 
-		sendBase:                97236,
+		sendBase:                29233,
-		sendTransferFunds:       96812,
+		sendTransferFunds:       27500,
-		sendTransferOnlyPremium: 347806,
+		sendTransferOnlyPremium: 159672,
-		sendInvokeMethod:        -3110,
+		sendInvokeMethod:        -5377,
 
-		ipldGetBase:    417230,
+		ipldGetBase:    75242,
-		ipldPutBase:    396100,
+		ipldPutBase:    84070,
 		ipldPutPerByte: 1,
 
-		createActorCompute: 750011,
+		createActorCompute: 1108454,
 		createActorStorage: 36 + 40,
 		deleteActor:        -(36 + 40), // -createActorStorage
 
 		verifySignature: map[crypto.SigType]int64{
-			crypto.SigTypeBLS:       219946580,
+			crypto.SigTypeBLS:       16598605,
-			crypto.SigTypeSecp256k1: 6726720,
+			crypto.SigTypeSecp256k1: 1637292,
 		},
 
-		hashingBase:                  110685,
+		hashingBase:                  31355,
-		computeUnsealedSectorCidBase: 431890,
+		computeUnsealedSectorCidBase: 98647,
 		verifySealBase:               2000, // TODO gas , it VerifySeal syscall is not used
 		verifyPostLookup: map[abi.RegisteredPoStProof]scalingCost{
 			abi.RegisteredPoStProof_StackedDrgWindow512MiBV1: {
-				flat:  106102820,
+				flat:  123861062,
-				scale: 10238878,
+				scale: 9226981,
 			},
 			abi.RegisteredPoStProof_StackedDrgWindow32GiBV1: {
-				flat:  1165718059,
+				flat:  748593537,
-				scale: 166657,
+				scale: 85639,
 			},
 			abi.RegisteredPoStProof_StackedDrgWindow64GiBV1: {
-				// TODO, for now the same as 32GiB
+				flat:  748593537,
-				flat:  1165718059,
+				scale: 85639,
-				scale: 166657,
 			},
 		},
 		verifyConsensusFault: 495422,

cli/paych.go

@@ -58,12 +58,20 @@ var paychGetCmd = &cli.Command{
 
 		ctx := ReqContext(cctx)
 
+		// Send a message to chain to create channel / add funds to existing
+		// channel
 		info, err := api.PaychGet(ctx, from, to, types.BigInt(amt))
 		if err != nil {
 			return err
 		}
 
-		fmt.Println(info.Channel.String())
+		// Wait for the message to be confirmed
+		chAddr, err := api.PaychGetWaitReady(ctx, info.ChannelMessage)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(chAddr)
 		return nil
 	},
 }

cli/wallet.go

@@ -215,6 +215,10 @@ var walletImport = &cli.Command{
 			Usage: "specify input format for key",
 			Value: "hex-lotus",
 		},
+		&cli.BoolFlag{
+			Name:  "as-default",
+			Usage: "import the given key as your new default key",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -287,6 +291,12 @@ var walletImport = &cli.Command{
 			return err
 		}
 
+		if cctx.Bool("as-default") {
+			if err := api.WalletSetDefault(ctx, addr); err != nil {
+				return fmt.Errorf("failed to set default key: %w", err)
+			}
+		}
+
 		fmt.Printf("imported key %s successfully!\n", addr)
 		return nil
 	},

cmd/lotus-seal-worker/main.go

@@ -110,6 +110,11 @@ var runCmd = &cli.Command{
 			Usage: "enable precommit1 (32G sectors: 1 core, 128GiB Memory)",
 			Value: true,
 		},
+		&cli.BoolFlag{
+			Name:  "unseal",
+			Usage: "enable unsealing (32G sectors: 1 core, 128GiB Memory)",
+			Value: true,
+		},
 		&cli.BoolFlag{
 			Name:  "precommit2",
 			Usage: "enable precommit2 (32G sectors: all cores, 96GiB Memory)",
@@ -202,6 +207,9 @@ var runCmd = &cli.Command{
 		if cctx.Bool("precommit1") {
 			taskTypes = append(taskTypes, sealtasks.TTPreCommit1)
 		}
+		if cctx.Bool("unseal") {
+			taskTypes = append(taskTypes, sealtasks.TTUnseal)
+		}
 		if cctx.Bool("precommit2") {
 			taskTypes = append(taskTypes, sealtasks.TTPreCommit2)
 		}

documentation/en/local-dev-net.md

@@ -22,13 +22,13 @@ Create the genesis block and start up the first node:
 ```sh
 ./lotus-seed genesis new localnet.json
 ./lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json
-./lotus daemon --lotus-make-genesis=dev.gen --genesis-template=localnet.json --bootstrap=false
+./lotus daemon --lotus-make-genesis=devgen.car --genesis-template=localnet.json --bootstrap=false
 ```
 
 Then, in another console, import the genesis miner key:
 
 ```sh
-./lotus wallet import ~/.genesis-sectors/pre-seal-t01000.key
+./lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key
 ```
 
 Set up the genesis miner:

markets/retrievaladapter/provider.go

@@ -11,6 +11,7 @@ import (
 	"github.com/filecoin-project/sector-storage/storiface"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/builtin/paych"
+	"github.com/ipfs/go-cid"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -57,7 +58,11 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi
 
 	r, w := io.Pipe()
 	go func() {
-		err := rpn.sealer.ReadPiece(ctx, w, sid, storiface.UnpaddedByteIndex(offset), length, si.TicketValue, *si.CommD)
+		var commD cid.Cid
+		if si.CommD != nil {
+			commD = *si.CommD
+		}
+		err := rpn.sealer.ReadPiece(ctx, w, sid, storiface.UnpaddedByteIndex(offset), length, si.TicketValue, commD)
 		_ = w.CloseWithError(err)
 	}()
 

node/impl/storminer.go

@@ -298,6 +298,24 @@ func (sm *StorageMinerAPI) MarketListRetrievalDeals(ctx context.Context) ([]retr
 	return out, nil
 }
 
+func (sm *StorageMinerAPI) MarketGetDealUpdates(ctx context.Context, d cid.Cid) (<-chan storagemarket.MinerDeal, error) {
+	results := make(chan storagemarket.MinerDeal)
+	unsub := sm.StorageProvider.SubscribeToEvents(func(evt storagemarket.ProviderEvent, deal storagemarket.MinerDeal) {
+		if deal.ProposalCid.Equals(d) {
+			select {
+			case results <- deal:
+			case <-ctx.Done():
+			}
+		}
+	})
+	go func() {
+		<-ctx.Done()
+		unsub()
+		close(results)
+	}()
+	return results, nil
+}
+
 func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) {
 	return sm.StorageProvider.ListLocalDeals()
 }

node/node_test.go

@@ -509,6 +509,9 @@ func TestAPIDealFlow(t *testing.T) {
 	t.Run("TestDoubleDealFlow", func(t *testing.T) {
 		test.TestDoubleDealFlow(t, mockSbBuilder, 10*time.Millisecond)
 	})
+	t.Run("TestFastRetrievalDealFlow", func(t *testing.T) {
+		test.TestFastRetrievalDealFlow(t, mockSbBuilder, 10*time.Millisecond)
+	})
 }
 
 func TestAPIDealFlowReal(t *testing.T) {

paychmgr/simple.go

@@ -403,6 +403,7 @@ func (ca *channelAccessor) mutateChannelInfo(channelID string, mutate func(*Chan
 	// the message, and then record that the message was sent.
 	if err != nil {
 		log.Errorf("Error reading channel info from store: %s", err)
+		return
 	}
 
 	mutate(channelInfo)