Merge pull request #11388 from filecoin-project/feat/lp-migrate2

cli: lotus-shed provider from-miner

cmd/lotus-provider/config.go

@@ -27,6 +27,7 @@ var configCmd = &cli.Command{
 		configListCmd,
 		configViewCmd,
 		configRmCmd,
+		configMigrateCmd,
 	},
 }
 
@@ -226,6 +227,10 @@ func getConfig(cctx *cli.Context, db *harmonydb.DB) (*config.LotusProviderConfig
 			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
 				return nil, fmt.Errorf("missing layer '%s' ", layer)
 			}
+			if layer == "base" {
+				return nil, errors.New(`lotus-provider defaults to a layer named 'base'. 
+				Either use 'migrate' command or edit a base.toml and upload it with: lotus-provider config set base.toml`)
+			}
 			return nil, fmt.Errorf("could not read layer '%s': %w", layer, err)
 		}
 		meta, err := toml.Decode(text, &lp)

cmd/lotus-provider/main.go

@@ -132,7 +132,7 @@ func main() {
 			&cli.StringFlag{
 				Name:    FlagRepoPath,
 				EnvVars: []string{"LOTUS_REPO_PATH"},
-				Value:   "~/.lotus",
+				Value:   "~/.lotusprovider",
 			},
 			cliutil.FlagVeryVerbose,
 		},

cmd/lotus-provider/migrate.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/samber/lo"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var configMigrateCmd = &cli.Command{
+	Name:        "from-miner",
+	Description: "Express a database config (for lotus-provider) from an existing miner.",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    FlagMinerRepo,
+			Aliases: []string{FlagMinerRepoDeprecation},
+			EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"},
+			Value:   "~/.lotusminer",
+			Usage:   fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation),
+		},
+		&cli.StringFlag{
+			Name:    "to-layer",
+			Aliases: []string{"t"},
+			Usage:   "The layer name for this data push. 'base' is recommended for single-miner setup.",
+		},
+		&cli.BoolFlag{
+			Name:    "replace",
+			Aliases: []string{"r"},
+			Usage:   "Use this with --to-layer to replace an existing layer",
+		},
+	},
+	Action: fromMiner,
+}
+
+const (
+	FlagMinerRepo = "miner-repo"
+)
+
+const FlagMinerRepoDeprecation = "storagerepo"
+
+func fromMiner(cctx *cli.Context) (err error) {
+	ctx := context.Background()
+
+	r, err := repo.NewFS(cctx.String(FlagMinerRepo))
+	if err != nil {
+		return err
+	}
+
+	ok, err := r.Exists()
+	if err != nil {
+		return err
+	}
+
+	if !ok {
+		return fmt.Errorf("repo not initialized")
+	}
+
+	lr, err := r.LockRO(repo.StorageMiner)
+	if err != nil {
+		return fmt.Errorf("locking repo: %w", err)
+	}
+	defer func() { _ = lr.Close() }()
+
+	cfgNode, err := lr.Config()
+	if err != nil {
+		return fmt.Errorf("getting node config: %w", err)
+	}
+	smCfg := cfgNode.(*config.StorageMiner)
+
+	db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB)
+	if err != nil {
+		return fmt.Errorf("could not reach the database. Ensure the Miner config toml's HarmonyDB entry"+
+			" is setup to reach Yugabyte correctly: %w", err)
+	}
+
+	var titles []string
+	err = db.Select(ctx, &titles, `SELECT title FROM harmony_config WHERE LENGTH(config) > 0`)
+	if err != nil {
+		return fmt.Errorf("miner cannot reach the db. Ensure the config toml's HarmonyDB entry"+
+			" is setup to reach Yugabyte correctly: %s", err.Error())
+	}
+	name := cctx.String("to-layer")
+	if name == "" {
+		name = fmt.Sprintf("mig%d", len(titles))
+	} else {
+		if lo.Contains(titles, name) && !cctx.Bool("overwrite") {
+			return errors.New("the overwrite flag is needed to replace existing layer: " + name)
+		}
+	}
+	msg := "Layer " + name + ` created. `
+
+	// Copy over identical settings:
+
+	buf, err := os.ReadFile(path.Join(lr.Path(), "config.toml"))
+	if err != nil {
+		return fmt.Errorf("could not read config.toml: %w", err)
+	}
+	var lpCfg config.LotusProviderConfig
+	_, err = toml.Decode(string(buf), &lpCfg)
+	if err != nil {
+		return fmt.Errorf("could not decode toml: %w", err)
+	}
+
+	// Populate Miner Address
+	sm, cc, err := cliutil.GetStorageMinerAPI(cctx)
+	if err != nil {
+		return fmt.Errorf("could not get storageMiner API: %w", err)
+	}
+	defer cc()
+	addr, err := sm.ActorAddress(ctx)
+	if err != nil {
+		return fmt.Errorf("could not read actor address: %w", err)
+	}
+
+	lpCfg.Addresses.MinerAddresses = []string{addr.String()}
+
+	ks, err := lr.KeyStore()
+	if err != nil {
+		return xerrors.Errorf("keystore err: %w", err)
+	}
+	js, err := ks.Get(modules.JWTSecretName)
+	if err != nil {
+		return xerrors.Errorf("error getting JWTSecretName: %w", err)
+	}
+	lpCfg.Apis.StorageRPCSecret = base64.RawStdEncoding.EncodeToString(js.PrivateKey)
+
+	// Populate API Key
+	_, header, err := cliutil.GetRawAPI(cctx, repo.FullNode, "v0")
+	if err != nil {
+		return fmt.Errorf("cannot read API: %w", err)
+	}
+
+	lpCfg.Apis.ChainApiInfo = []string{header.Get("Authorization")[7:]}
+
+	// Enable WindowPoSt
+	lpCfg.Subsystems.EnableWindowPost = true
+	msg += `\nBefore running lotus-provider, ensure any miner/worker answering of WindowPost is disabled by
+(on Miner) DisableBuiltinWindowPoSt=true and (on Workers) not enabling windowpost on CLI or via
+environment variable LOTUS_WORKER_WINDOWPOST.
+`
+
+	// Express as configTOML
+	configTOML := &bytes.Buffer{}
+	if err = toml.NewEncoder(configTOML).Encode(lpCfg); err != nil {
+		return err
+	}
+
+	if !lo.Contains(titles, "base") {
+		_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', '')", "base")
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", name, configTOML.String())
+	if err != nil {
+		return err
+	}
+
+	dbSettings := ""
+	def := config.DefaultStorageMiner().HarmonyDB
+	if def.Hosts[0] != smCfg.HarmonyDB.Hosts[0] {
+		dbSettings += ` --db-host="` + strings.Join(smCfg.HarmonyDB.Hosts, ",") + `"`
+	}
+	if def.Port != smCfg.HarmonyDB.Port {
+		dbSettings += " --db-port=" + smCfg.HarmonyDB.Port
+	}
+	if def.Username != smCfg.HarmonyDB.Username {
+		dbSettings += ` --db-user="` + smCfg.HarmonyDB.Username + `"`
+	}
+	if def.Password != smCfg.HarmonyDB.Password {
+		dbSettings += ` --db-password="` + smCfg.HarmonyDB.Password + `"`
+	}
+	if def.Database != smCfg.HarmonyDB.Database {
+		dbSettings += ` --db-name="` + smCfg.HarmonyDB.Database + `"`
+	}
+
+	msg += `
+To work with the config:
+./lotus-provider ` + dbSettings + ` config help `
+	msg += `
+To run Lotus Provider: in its own machine or cgroup without other files, use the command: 
+./lotus-provider ` + dbSettings + ` run --layers="` + name + `"
+	`
+	fmt.Println(msg)
+	return nil
+}

cmd/lotus-provider/run.go

@@ -196,7 +196,7 @@ var runCmd = &cli.Command{
 		}
 		defer j.Close()
 
-		full, fullCloser, err := cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfg.Apis.FULLNODE_API_INFO)
+		full, fullCloser, err := cliutil.GetFullNodeAPIV1LotusProvider(cctx, cfg.Apis.ChainApiInfo)
 		if err != nil {
 			return err
 		}

node/config/types.go

@@ -77,8 +77,8 @@ type LotusProviderConfig struct {
 }
 
 type ApisConfig struct {
-	// FULLNODE_API_INFO is the API endpoint for the Lotus daemon.
-	FULLNODE_API_INFO []string
+	// ChainApiInfo is the API endpoint for the Lotus daemon.
+	ChainApiInfo []string
 
 	// RPC Secret for the storage subsystem.
 	// If integrating with lotus-miner this must match the value from