From d18cb5d51e09ce3a82cebed4732d0278f1afbd60 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 6 Nov 2023 22:49:42 -0600 Subject: [PATCH] cli: lotus-shed provider from-miner --- cmd/lotus-provider/main.go | 2 +- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/provider.go | 180 +++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 cmd/lotus-shed/provider.go diff --git a/cmd/lotus-provider/main.go b/cmd/lotus-provider/main.go index c800a9650..a7b260402 100644 --- a/cmd/lotus-provider/main.go +++ b/cmd/lotus-provider/main.go @@ -132,7 +132,7 @@ func main() { &cli.StringFlag{ Name: FlagRepoPath, EnvVars: []string{"LOTUS_REPO_PATH"}, - Value: "~/.lotus", + Value: "~/.lotusprovider", }, cliutil.FlagVeryVerbose, }, diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index a5b66a096..4ff3fc2aa 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -92,6 +92,7 @@ func main() { FevmAnalyticsCmd, mismatchesCmd, blockCmd, + providerCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/provider.go b/cmd/lotus-shed/provider.go new file mode 100644 index 000000000..a9da4695e --- /dev/null +++ b/cmd/lotus-shed/provider.go @@ -0,0 +1,180 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "strings" + + "github.com/BurntSushi/toml" + cliutil "github.com/filecoin-project/lotus/cli/util" + "github.com/filecoin-project/lotus/lib/harmony/harmonydb" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/repo" + "github.com/samber/lo" + "github.com/urfave/cli/v2" +) + +var providerCmd = &cli.Command{ + Name: "provider", + Description: "Run a lotus-provider helper", + Subcommands: []*cli.Command{ + { + Name: "from-miner", + Description: "Express a database config from an existing miner.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagMinerRepo, + Aliases: []string{FlagMinerRepoDeprecation}, + EnvVars: []string{"LOTUS_MINER_PATH", "LOTUS_STORAGE_PATH"}, + Value: "~/.lotusminer", + Usage: fmt.Sprintf("Specify miner repo path. flag(%s) and env(LOTUS_STORAGE_PATH) are DEPRECATION, will REMOVE SOON", FlagMinerRepoDeprecation), + }, + &cli.StringFlag{ + Name: "layer", + Aliases: []string{"l"}, + Usage: "The layer name for this data push. 'base' is recommended for single-miner setup. Overwrites.", + }, + }, + Action: fromMiner, + }, + }, +} + +const ( + FlagMinerRepo = "miner-repo" +) + +const FlagMinerRepoDeprecation = "storagerepo" + +func fromMiner(cctx *cli.Context) error { + ctx := context.Background() + + r, err := repo.NewFS(cctx.String(FlagMinerRepo)) + if err != nil { + return err + } + + ok, err := r.Exists() + if err != nil { + return err + } + + if !ok { + return fmt.Errorf("repo not initialized") + } + + lr, err := r.LockRO(repo.StorageMiner) + if err != nil { + return fmt.Errorf("locking repo: %w", err) + } + defer func() { _ = lr.Close() }() + + cfgNode, err := lr.Config() + if err != nil { + return fmt.Errorf("getting node config: %w", err) + } + smCfg := cfgNode.(*config.StorageMiner) + + db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB) + if err != nil { + return fmt.Errorf("Could not reach the database. Ensure the Miner config toml's HarmonyDB entry" + + " is setup to reach Yugabyte correctly.") + } + + var titles []string + err = db.Select(ctx, &titles, `SELECT title FROM harmony_config`) + if err != nil { + return fmt.Errorf("miner cannot reach the db. Ensure the config toml's HarmonyDB entry"+ + " is setup to reach Yugabyte correctly: %s", err.Error()) + } + name := cctx.String("layer") + if name == "" { + name = fmt.Sprintf("mig%d", len(titles)) + } + msg := "Layer " + name + ` created. ` + + // Step 1: copy over identical settings: + buf := &bytes.Buffer{} + if err = toml.NewEncoder(buf).Encode(smCfg); err != nil { + return err + } + var lpCfg config.LotusProviderConfig + _, err = toml.Decode(buf.String(), &lpCfg) + if err != nil { + return fmt.Errorf("could not decode toml: %w", err) + } + + // Populate Miner Address + sm, cc, err := cliutil.GetStorageMinerAPI(cctx) + if err != nil { + return fmt.Errorf("could not get storageMiner API: %w", err) + } + defer cc() + addr, err := sm.ActorAddress(ctx) + if err != nil { + return fmt.Errorf("could not read actor address: %w", err) + } + + lpCfg.Addresses.MinerAddresses = []string{addr.String()} + + // TODO Apis.StorageSecret storage repo and reading the rpc secret + //lr.?? + + // Populate API Key + _, header, err := cliutil.GetRawAPI(cctx, repo.FullNode, "v0") + if err != nil { + return fmt.Errorf("Cannot read API: %w", err) + } + + lpCfg.Apis.FULLNODE_API_INFO = []string{header.Get("Authorization")[7:]} + + // Step 4: Enable WindowPoSt + lpCfg.Subsystems.EnableWindowPost = true + msg += `\nBefore running lotus-provider, ensure any miner/worker answering of WindowPost is disabled.\n` + + // Step 5 Express as configTOML + configTOML := &bytes.Buffer{} + if err = toml.NewEncoder(buf).Encode(lpCfg); err != nil { + return err + } + + if !lo.Contains(titles, "base") { + _, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ('base', '')", "base") + if err != nil { + return err + } + } + _, err = db.Exec(ctx, "INSERT INTO harmony_config (title, config) VALUES ($1, $2)", name, configTOML.String()) + if err != nil { + return err + } + + dbSettings := "" + def := config.DefaultStorageMiner().HarmonyDB + if def.Hosts[0] != smCfg.HarmonyDB.Hosts[0] { + dbSettings += ` --db-host="` + strings.Join(smCfg.HarmonyDB.Hosts, ",") + `"` + } + if def.Port != smCfg.HarmonyDB.Port { + dbSettings += " --db-port=" + smCfg.HarmonyDB.Port + } + if def.Username != smCfg.HarmonyDB.Username { + dbSettings += ` --db-user="` + smCfg.HarmonyDB.Username + `"` + } + if def.Password != smCfg.HarmonyDB.Password { + dbSettings += ` --db-password="` + smCfg.HarmonyDB.Password + `"` + } + if def.Database != smCfg.HarmonyDB.Database { + dbSettings += ` --db-name="` + smCfg.HarmonyDB.Database + `"` + } + + msg += ` +To work with the config: +./lotus-provider ` + dbSettings + ` config help ` + msg += ` +To run Lotus Provider: in its own machine or cgroup without other files, use the command: +./lotus-provider ` + dbSettings + ` run --layers="` + name + `" + ` + fmt.Println(msg) + return nil +}