// guidedSetup for migration from lotus-miner to Curio // // IF STRINGS CHANGED { // follow instructions at ../internal/translations/translations.go // } package guidedsetup import ( "bytes" "context" "encoding/base64" "encoding/json" "fmt" "io" "math/bits" "net/http" "os" "os/signal" "path" "reflect" "strings" "syscall" "time" "github.com/BurntSushi/toml" "github.com/charmbracelet/lipgloss" "github.com/manifoldco/promptui" "github.com/mitchellh/go-homedir" "github.com/samber/lo" "github.com/urfave/cli/v2" "golang.org/x/text/language" "golang.org/x/text/message" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v0api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" cliutil "github.com/filecoin-project/lotus/cli/util" _ "github.com/filecoin-project/lotus/cmd/curio/internal/translations" "github.com/filecoin-project/lotus/lib/harmony/harmonydb" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) // URL to upload user-selected fields to help direct developer's focus. const DeveloperFocusRequestURL = "https://curiostorage.org/cgi-bin/savedata.php" var GuidedsetupCmd = &cli.Command{ Name: "guided-setup", Usage: "Run the guided setup for migrating from lotus-miner to Curio", Flags: []cli.Flag{ &cli.StringFlag{ // for cliutil.GetFullNodeAPI Name: "repo", EnvVars: []string{"LOTUS_PATH"}, Hidden: true, Value: "~/.lotus", }, }, Action: func(cctx *cli.Context) (err error) { T, say := SetupLanguage() setupCtrlC(say) say(header, "This interactive tool migrates lotus-miner to Curio in 5 minutes.") say(notice, "Each step needs your confirmation and can be reversed. Press Ctrl+C to exit at any time.") // Run the migration steps migrationData := MigrationData{ T: T, say: say, selectTemplates: &promptui.SelectTemplates{ Help: T("Use the arrow keys to navigate: ↓ ↑ → ← "), }, cctx: cctx, } for _, step := range migrationSteps { step(&migrationData) } for _, closer := range migrationData.closers { closer() } return nil }, } func setupCtrlC(say func(style lipgloss.Style, key message.Reference, a ...interface{})) { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { <-c say(notice, "Ctrl+C pressed in Terminal") os.Exit(2) }() } var ( header = lipgloss.NewStyle(). Align(lipgloss.Left). Foreground(lipgloss.Color("#00FF00")). Background(lipgloss.Color("#242424")). BorderStyle(lipgloss.NormalBorder()). Width(60).Margin(1) notice = lipgloss.NewStyle(). Align(lipgloss.Left). Bold(true). Foreground(lipgloss.Color("#CCCCCC")). Background(lipgloss.Color("#333300")).MarginBottom(1) green = lipgloss.NewStyle(). Align(lipgloss.Left). Foreground(lipgloss.Color("#00FF00")). Background(lipgloss.Color("#000000")) plain = lipgloss.NewStyle().Align(lipgloss.Left) section = lipgloss.NewStyle(). Align(lipgloss.Left). Foreground(lipgloss.Color("#000000")). Background(lipgloss.Color("#FFFFFF")). Underline(true) code = lipgloss.NewStyle(). Align(lipgloss.Left). Foreground(lipgloss.Color("#00FF00")). Background(lipgloss.Color("#f8f9fa")) ) func SetupLanguage() (func(key message.Reference, a ...interface{}) string, func(style lipgloss.Style, key message.Reference, a ...interface{})) { langText := "en" problem := false if len(os.Getenv("LANG")) > 1 { langText = os.Getenv("LANG")[:2] } else { problem = true } lang, err := language.Parse(langText) if err != nil { lang = language.English problem = true fmt.Println("Error parsing language") } langs := message.DefaultCatalog.Languages() have := lo.SliceToMap(langs, func(t language.Tag) (string, bool) { return t.String(), true }) if _, ok := have[lang.String()]; !ok { lang = language.English problem = true } if problem { _ = os.Setenv("LANG", "en-US") // for later users of this function notice.Copy().AlignHorizontal(lipgloss.Right). Render("$LANG=" + langText + " unsupported. Available: " + strings.Join(lo.Keys(have), ", ")) fmt.Println("Defaulting to English. Please reach out to the Curio team if you would like to have additional language support.") } return func(key message.Reference, a ...interface{}) string { return message.NewPrinter(lang).Sprintf(key, a...) }, func(sty lipgloss.Style, key message.Reference, a ...interface{}) { msg := message.NewPrinter(lang).Sprintf(key, a...) fmt.Println(sty.Render(msg)) } } type migrationStep func(*MigrationData) var migrationSteps = []migrationStep{ readMinerConfig, // Tells them to be on the miner machine yugabyteConnect, // Miner is updated configToDB, // work on base configuration migration. verifySectors, // Verify the sectors are in the database doc, oneLastThing, complete, } type MigrationData struct { T func(key message.Reference, a ...interface{}) string say func(style lipgloss.Style, key message.Reference, a ...interface{}) selectTemplates *promptui.SelectTemplates MinerConfigPath string MinerConfig *config.StorageMiner DB *harmonydb.DB MinerID address.Address full v0api.FullNode cctx *cli.Context closers []jsonrpc.ClientCloser } func complete(d *MigrationData) { stepCompleted(d, d.T("Lotus-Miner to Curio Migration.")) d.say(plain, "Try the web interface with %s for further guided improvements.", "--layers=gui") d.say(plain, "You can now migrate your market node (%s), if applicable.", "Boost") } func configToDB(d *MigrationData) { d.say(section, "Migrating lotus-miner config.toml to Curio in-database configuration.") { var closer jsonrpc.ClientCloser var err error d.full, closer, err = cliutil.GetFullNodeAPI(d.cctx) d.closers = append(d.closers, closer) if err != nil { d.say(notice, "Error getting API: %s", err.Error()) os.Exit(1) } } ainfo, err := cliutil.GetAPIInfo(d.cctx, repo.FullNode) if err != nil { d.say(notice, "could not get API info for FullNode: %w", err) os.Exit(1) } token, err := d.full.AuthNew(context.Background(), api.AllPermissions) if err != nil { d.say(notice, "Error getting token: %s", err.Error()) os.Exit(1) } chainApiInfo := fmt.Sprintf("%s:%s", string(token), ainfo.Addr) d.MinerID, err = SaveConfigToLayer(d.MinerConfigPath, "", false, chainApiInfo) if err != nil { d.say(notice, "Error saving config to layer: %s. Aborting Migration", err.Error()) os.Exit(1) } } // bucket returns the power's 4 highest bits (rounded down). func bucket(power *api.MinerPower) uint64 { rawQAP := power.TotalPower.QualityAdjPower.Uint64() magnitude := lo.Max([]int{bits.Len64(rawQAP), 5}) // shifting erases resolution so we cannot distinguish SPs of similar scales. return rawQAP >> (uint64(magnitude) - 4) << (uint64(magnitude - 4)) } type uploadType int const uploadTypeIndividual uploadType = 0 const uploadTypeAggregate uploadType = 1 // const uploadTypeHint uploadType = 2 const uploadTypeNothing uploadType = 3 func oneLastThing(d *MigrationData) { d.say(section, "The Curio team wants to improve the software you use. Tell the team you're using `%s`.", "curio") i, _, err := (&promptui.Select{ Label: d.T("Select what you want to share with the Curio team."), Items: []string{ d.T("Individual Data: Miner ID, Curio version, chain (%s or %s). Signed.", "mainnet", "calibration"), d.T("Aggregate-Anonymous: version, chain, and Miner power (bucketed)."), d.T("Hint: I am someone running Curio on whichever chain."), d.T("Nothing.")}, Templates: d.selectTemplates, }).Run() preference := uploadType(i) if err != nil { d.say(notice, "Aborting remaining steps.", err.Error()) os.Exit(1) } if preference != uploadTypeNothing { msgMap := map[string]any{ "domain": "curio-newuser", "net": build.BuildTypeString(), } if preference == uploadTypeIndividual || preference == uploadTypeAggregate { // articles of incorporation power, err := d.full.StateMinerPower(context.Background(), d.MinerID, types.EmptyTSK) if err != nil { d.say(notice, "Error getting miner power: %s", err.Error()) os.Exit(1) } msgMap["version"] = build.BuildVersion msgMap["net"] = build.BuildType msgMap["power"] = map[uploadType]uint64{ uploadTypeIndividual: power.MinerPower.QualityAdjPower.Uint64(), uploadTypeAggregate: bucket(power)}[preference] if preference == uploadTypeIndividual { // Sign it msgMap["miner_id"] = d.MinerID msg, err := json.Marshal(msgMap) if err != nil { d.say(notice, "Error marshalling message: %s", err.Error()) os.Exit(1) } mi, err := d.full.StateMinerInfo(context.Background(), d.MinerID, types.EmptyTSK) if err != nil { d.say(notice, "Error getting miner info: %s", err.Error()) os.Exit(1) } sig, err := d.full.WalletSign(context.Background(), mi.Worker, msg) if err != nil { d.say(notice, "Error signing message: %s", err.Error()) os.Exit(1) } msgMap["signature"] = base64.StdEncoding.EncodeToString(sig.Data) } } msg, err := json.Marshal(msgMap) if err != nil { d.say(notice, "Error marshalling message: %s", err.Error()) os.Exit(1) } resp, err := http.DefaultClient.Post(DeveloperFocusRequestURL, "application/json", bytes.NewReader(msg)) if err != nil { d.say(notice, "Error sending message: %s", err.Error()) } if resp != nil { defer func() { _ = resp.Body.Close() }() if resp.StatusCode != 200 { b, err := io.ReadAll(resp.Body) if err == nil { d.say(notice, "Error sending message: Status %s, Message: ", resp.Status, string(b)) } } else { stepCompleted(d, d.T("Message sent.")) } } } } func doc(d *MigrationData) { d.say(plain, "Documentation: ") d.say(plain, "The '%s' layer stores common configuration. All curio instances can include it in their %s argument.", "base", "--layers") d.say(plain, "You can add other layers for per-machine configuration changes.") d.say(plain, "Filecoin %s channels: %s and %s", "Slack", "#fil-curio-help", "#fil-curio-dev") d.say(plain, "Start multiple Curio instances with the '%s' layer to redundancy.", "post") //d.say(plain, "Point your browser to your web GUI to complete setup with %s and advanced featues.", "Boost") d.say(plain, "One database can serve multiple miner IDs: Run a migration for each lotus-miner.") } func verifySectors(d *MigrationData) { var i []int var lastError string fmt.Println() d.say(section, "Please start (or restart) %s now that database credentials are in %s.", "lotus-miner", "config.toml") d.say(notice, "Waiting for %s to write sectors into Yugabyte.", "lotus-miner") mid, err := address.IDFromAddress(d.MinerID) if err != nil { d.say(notice, "Error interpreting miner ID: %s: ID: %s", err.Error(), d.MinerID.String()) os.Exit(1) } for { err := d.DB.Select(context.Background(), &i, ` SELECT count(*) FROM sector_location WHERE miner_id=$1`, mid) if err != nil { if err.Error() != lastError { d.say(notice, "Error verifying sectors: %s", err.Error()) lastError = err.Error() } continue } if i[0] > 0 { break } fmt.Print(".") time.Sleep(5 * time.Second) } d.say(plain, "The sectors are in the database. The database is ready for %s.", "Curio") d.say(notice, "Now shut down lotus-miner and move the systems to %s.", "Curio") _, err = (&promptui.Prompt{Label: d.T("Press return to continue")}).Run() if err != nil { d.say(notice, "Aborting migration.") os.Exit(1) } stepCompleted(d, d.T("Sectors verified. %d sector locations found.", i)) } func yugabyteConnect(d *MigrationData) { harmonyCfg := config.DefaultStorageMiner().HarmonyDB //copy the config to a local variable if d.MinerConfig != nil { harmonyCfg = d.MinerConfig.HarmonyDB //copy the config to a local variable } var err error d.DB, err = harmonydb.NewFromConfig(harmonyCfg) if err == nil { goto yugabyteConnected } for { i, _, err := (&promptui.Select{ Label: d.T("Enter the info to connect to your Yugabyte database installation (https://download.yugabyte.com/)"), Items: []string{ d.T("Host: %s", strings.Join(harmonyCfg.Hosts, ",")), d.T("Port: %s", harmonyCfg.Port), d.T("Username: %s", harmonyCfg.Username), d.T("Password: %s", harmonyCfg.Password), d.T("Database: %s", harmonyCfg.Database), d.T("Continue to connect and update schema.")}, Size: 6, Templates: d.selectTemplates, }).Run() if err != nil { d.say(notice, "Database config error occurred, abandoning migration: %s ", err.Error()) os.Exit(1) } switch i { case 0: host, err := (&promptui.Prompt{ Label: d.T("Enter the Yugabyte database host(s)"), }).Run() if err != nil { d.say(notice, "No host provided") continue } harmonyCfg.Hosts = strings.Split(host, ",") case 1, 2, 3, 4: val, err := (&promptui.Prompt{ Label: d.T("Enter the Yugabyte database %s", []string{"port", "username", "password", "database"}[i-1]), }).Run() if err != nil { d.say(notice, "No value provided") continue } switch i { case 1: harmonyCfg.Port = val case 2: harmonyCfg.Username = val case 3: harmonyCfg.Password = val case 4: harmonyCfg.Database = val } continue case 5: d.DB, err = harmonydb.NewFromConfig(harmonyCfg) if err != nil { if err.Error() == "^C" { os.Exit(1) } d.say(notice, "Error connecting to Yugabyte database: %s", err.Error()) continue } goto yugabyteConnected } } yugabyteConnected: d.say(plain, "Connected to Yugabyte. Schema is current.") if !reflect.DeepEqual(harmonyCfg, d.MinerConfig.HarmonyDB) || !d.MinerConfig.Subsystems.EnableSectorIndexDB { d.MinerConfig.HarmonyDB = harmonyCfg d.MinerConfig.Subsystems.EnableSectorIndexDB = true d.say(plain, "Enabling Sector Indexing in the database.") buf, err := config.ConfigUpdate(d.MinerConfig, config.DefaultStorageMiner()) if err != nil { d.say(notice, "Error encoding config.toml: %s", err.Error()) os.Exit(1) } _, err = (&promptui.Prompt{ Label: d.T("Press return to update %s with Yugabyte info. A Backup file will be written to that folder before changes are made.", "config.toml")}).Run() if err != nil { os.Exit(1) } p, err := homedir.Expand(d.MinerConfigPath) if err != nil { d.say(notice, "Error expanding path: %s", err.Error()) os.Exit(1) } tomlPath := path.Join(p, "config.toml") stat, err := os.Stat(tomlPath) if err != nil { d.say(notice, "Error reading filemode of config.toml: %s", err.Error()) os.Exit(1) } fBackup, err := os.CreateTemp(p, "config-backup-*.toml") if err != nil { d.say(notice, "Error creating backup file: %s", err.Error()) os.Exit(1) } fBackupContents, err := os.ReadFile(tomlPath) if err != nil { d.say(notice, "Error reading config.toml: %s", err.Error()) os.Exit(1) } _, err = fBackup.Write(fBackupContents) if err != nil { d.say(notice, "Error writing backup file: %s", err.Error()) os.Exit(1) } err = fBackup.Close() if err != nil { d.say(notice, "Error closing backup file: %s", err.Error()) os.Exit(1) } filemode := stat.Mode() err = os.WriteFile(path.Join(p, "config.toml"), buf, filemode) if err != nil { d.say(notice, "Error writing config.toml: %s", err.Error()) os.Exit(1) } d.say(section, "Restart Lotus Miner. ") } stepCompleted(d, d.T("Connected to Yugabyte")) } func readMinerConfig(d *MigrationData) { d.say(plain, "To start, ensure your sealing pipeline is drained and shut-down lotus-miner.") verifyPath := func(dir string) (*config.StorageMiner, error) { cfg := config.DefaultStorageMiner() dir, err := homedir.Expand(dir) if err != nil { return nil, err } _, err = toml.DecodeFile(path.Join(dir, "config.toml"), &cfg) return cfg, err } dirs := map[string]*config.StorageMiner{"~/.lotusminer": nil, "~/.lotus-miner-local-net": nil} if v := os.Getenv("LOTUS_MINER_PATH"); v != "" { dirs[v] = nil } for dir := range dirs { cfg, err := verifyPath(dir) if err != nil { delete(dirs, dir) } dirs[dir] = cfg } var otherPath bool if len(dirs) > 0 { _, str, err := (&promptui.Select{ Label: d.T("Select the location of your lotus-miner config directory?"), Items: append(lo.Keys(dirs), d.T("Other")), Templates: d.selectTemplates, }).Run() if err != nil { if err.Error() == "^C" { os.Exit(1) } otherPath = true } else { if str == d.T("Other") { otherPath = true } else { d.MinerConfigPath = str d.MinerConfig = dirs[str] } } } if otherPath { minerPathEntry: str, err := (&promptui.Prompt{ Label: d.T("Enter the path to the configuration directory used by %s", "lotus-miner"), }).Run() if err != nil { d.say(notice, "No path provided, abandoning migration ") os.Exit(1) } cfg, err := verifyPath(str) if err != nil { d.say(notice, "Cannot read the config.toml file in the provided directory, Error: %s", err.Error()) goto minerPathEntry } d.MinerConfigPath = str d.MinerConfig = cfg } // Try to lock Miner repo to verify that lotus-miner is not running { r, err := repo.NewFS(d.MinerConfigPath) if err != nil { d.say(plain, "Could not create repo from directory: %s. Aborting migration", err.Error()) os.Exit(1) } lr, err := r.Lock(repo.StorageMiner) if err != nil { d.say(plain, "Could not lock miner repo. Your miner must be stopped: %s\n Aborting migration", err.Error()) os.Exit(1) } _ = lr.Close() } stepCompleted(d, d.T("Read Miner Config")) } func stepCompleted(d *MigrationData, step string) { fmt.Print(green.Render("✔ ")) d.say(plain, "Step Complete: %s\n", step) }