From adba595350f1847c5831b5e98b5dccd3e4a8d67a Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 22 Jul 2021 18:41:57 +0200 Subject: [PATCH 1/4] commit from @dirkmc - initial export cmd for martkers related metadata --- cmd/lotus-shed/market.go | 136 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index e2e322784..cde66191d 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -2,6 +2,16 @@ package main import ( "fmt" + "io/ioutil" + "os" + "path" + + "github.com/filecoin-project/lotus/lib/backupds" + + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + logging "github.com/ipfs/go-log/v2" lcli "github.com/filecoin-project/lotus/cli" @@ -18,6 +28,7 @@ var marketCmd = &cli.Command{ Flags: []cli.Flag{}, Subcommands: []*cli.Command{ marketDealFeesCmd, + marketExportDatastoreCmd, }, } @@ -100,3 +111,128 @@ var marketDealFeesCmd = &cli.Command{ return xerrors.New("must provide either --provider or --dealId flag") }, } + +var marketExportDatastoreCmd = &cli.Command{ + Name: "export-datastore", + Description: "export datastore to a file", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Usage: "path to the repo", + }, + &cli.StringFlag{ + Name: "backup-dir", + Usage: "path to the backup directory", + }, + }, + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + backupDir := cctx.String("backup-dir") + if backupDir == "" { + backupDir = os.TempDir() + } + + r, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("opening fs repo: %w", err) + } + + exists, err := r.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lr, err := r.Lock(repo.StorageMiner) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + namespace := "metadata" + ds, err := lr.Datastore(cctx.Context, datastore.NewKey(namespace).String()) + if err != nil { + return err + } + + backupRepoDir, err := ioutil.TempDir("", "backup-repo-dir") + if err != nil { + return err + } + + backupRepo, err := repo.NewFS(cctx.String(backupRepoDir)) + if err != nil { + return xerrors.Errorf("opening backup repo: %w", err) + } + + lockedBackupRepo, err := backupRepo.Lock(repo.StorageMiner) + if err != nil { + return err + } + defer lockedBackupRepo.Close() //nolint:errcheck + + backupDs, err := lockedBackupRepo.Datastore(cctx.Context, datastore.NewKey(namespace).String()) + if err != nil { + return err + } + + prefixes := []string{ + "/deals/provider", + "/retrievals/provider", + "/storagemarket", + } + for _, prefix := range prefixes { + err := exportPrefix(prefix, ds, backupDs) + if err != nil { + return err + } + } + + bds, err := backupds.Wrap(backupDs, "") + if err != nil { + return xerrors.Errorf("opening backupds: %w", err) + } + + fpath := path.Join(backupDir, "datastore.backup") + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return xerrors.Errorf("open %s: %w", fpath, err) + } + if err := bds.Backup(out); err != nil { + if cerr := out.Close(); cerr != nil { + log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) + } + return xerrors.Errorf("backup error: %w", err) + } + if err := out.Close(); err != nil { + return xerrors.Errorf("closing backup file: %w", err) + } + + fmt.Println("Wrote backup file to " + fpath) + + return nil + }, +} + +func exportPrefix(prefix string, ds datastore.Batching, backupDs datastore.Batching) error { + q, err := ds.Query(dsq.Query{ + Prefix: prefix, + }) + if err != nil { + return xerrors.Errorf("datastore query: %w", err) + } + defer q.Close() //nolint:errcheck + + for res := range q.Next() { + fmt.Println("Exporting key " + res.Key) + err := backupDs.Put(datastore.NewKey(res.Key), res.Value) + if err != nil { + return xerrors.Errorf("putting %s to backup datastore: %w", res.Key, err) + } + } + + return nil +} From 7ef167b04f719941ac1bbf91faa9d9b932dfd5f0 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 23 Jul 2021 09:33:26 +0200 Subject: [PATCH 2/4] refactor: simplify market datastore backup --- cmd/lotus-shed/market.go | 44 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index cde66191d..10d71bbf2 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -2,10 +2,12 @@ package main import ( "fmt" - "io/ioutil" "os" "path" + levelds "github.com/ipfs/go-ds-leveldb" + ldbopts "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" @@ -128,16 +130,19 @@ var marketExportDatastoreCmd = &cli.Command{ Action: func(cctx *cli.Context) error { logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + // If the backup dir is not specified, just use the OS temp dir backupDir := cctx.String("backup-dir") if backupDir == "" { backupDir = os.TempDir() } + // Create a new repo at the repo path r, err := repo.NewFS(cctx.String("repo")) if err != nil { return xerrors.Errorf("opening fs repo: %w", err) } + // Make sure the repo exists exists, err := r.Exists() if err != nil { return err @@ -146,39 +151,30 @@ var marketExportDatastoreCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } + // Lock the repo lr, err := r.Lock(repo.StorageMiner) if err != nil { return err } defer lr.Close() //nolint:errcheck + // Open the metadata datastore on the repo namespace := "metadata" ds, err := lr.Datastore(cctx.Context, datastore.NewKey(namespace).String()) if err != nil { return err } - backupRepoDir, err := ioutil.TempDir("", "backup-repo-dir") - if err != nil { - return err - } - - backupRepo, err := repo.NewFS(cctx.String(backupRepoDir)) - if err != nil { - return xerrors.Errorf("opening backup repo: %w", err) - } - - lockedBackupRepo, err := backupRepo.Lock(repo.StorageMiner) - if err != nil { - return err - } - defer lockedBackupRepo.Close() //nolint:errcheck - - backupDs, err := lockedBackupRepo.Datastore(cctx.Context, datastore.NewKey(namespace).String()) - if err != nil { - return err - } + // Create a tmp datastore that we'll add the exported key / values to + // and then backup + backupDs, err := levelds.NewDatastore(backupDir, &levelds.Options{ + Compression: ldbopts.NoCompression, + NoSync: false, + Strict: ldbopts.StrictAll, + ReadOnly: false, + }) + // Export the key / values prefixes := []string{ "/deals/provider", "/retrievals/provider", @@ -191,16 +187,20 @@ var marketExportDatastoreCmd = &cli.Command{ } } + // Wrap the datastore in a backup datastore bds, err := backupds.Wrap(backupDs, "") if err != nil { return xerrors.Errorf("opening backupds: %w", err) } - fpath := path.Join(backupDir, "datastore.backup") + // Create a file for the backup + fpath := path.Join(backupDir, "markets.datastore.backup") out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return xerrors.Errorf("open %s: %w", fpath, err) } + + // Write the backup to the file if err := bds.Backup(out); err != nil { if cerr := out.Close(); cerr != nil { log.Errorw("error closing backup file while handling backup error", "closeErr", cerr, "backupErr", err) From 52498f78a1da96e8f9a62c2e6ce9a354c556b7ef Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Fri, 23 Jul 2021 10:30:58 +0200 Subject: [PATCH 3/4] feat: add import market datastore cmd --- cmd/lotus-shed/market.go | 117 +++++++++++++++++++++++++++++++-------- 1 file changed, 94 insertions(+), 23 deletions(-) diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index 10d71bbf2..eb1b56a94 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -31,6 +31,7 @@ var marketCmd = &cli.Command{ Subcommands: []*cli.Command{ marketDealFeesCmd, marketExportDatastoreCmd, + marketImportDatastoreCmd, }, } @@ -114,9 +115,11 @@ var marketDealFeesCmd = &cli.Command{ }, } +const mktsMetadataNamespace = "metadata" + var marketExportDatastoreCmd = &cli.Command{ Name: "export-datastore", - Description: "export datastore to a file", + Description: "export markets datastore key/values to a file", Flags: []cli.Flag{ &cli.StringFlag{ Name: "repo", @@ -136,43 +139,37 @@ var marketExportDatastoreCmd = &cli.Command{ backupDir = os.TempDir() } - // Create a new repo at the repo path - r, err := repo.NewFS(cctx.String("repo")) - if err != nil { - return xerrors.Errorf("opening fs repo: %w", err) - } - - // Make sure the repo exists - exists, err := r.Exists() - if err != nil { - return err - } - if !exists { - return xerrors.Errorf("lotus repo doesn't exist") - } - - // Lock the repo - lr, err := r.Lock(repo.StorageMiner) + // Open the repo at the repo path + repoPath := cctx.String("repo") + lr, err := openLockedRepo(repoPath) if err != nil { return err } defer lr.Close() //nolint:errcheck // Open the metadata datastore on the repo - namespace := "metadata" - ds, err := lr.Datastore(cctx.Context, datastore.NewKey(namespace).String()) + ds, err := lr.Datastore(cctx.Context, datastore.NewKey(mktsMetadataNamespace).String()) if err != nil { - return err + return xerrors.Errorf("opening datastore %s on repo %s: %w", mktsMetadataNamespace, repoPath, err) } // Create a tmp datastore that we'll add the exported key / values to // and then backup - backupDs, err := levelds.NewDatastore(backupDir, &levelds.Options{ + backupDsDir := path.Join(backupDir, "markets-backup-datastore") + if err := os.MkdirAll(backupDsDir, 0775); err != nil { //nolint:gosec + return xerrors.Errorf("creating tmp datastore directory: %w", err) + } + defer os.RemoveAll(backupDsDir) //nolint:errcheck + + backupDs, err := levelds.NewDatastore(backupDsDir, &levelds.Options{ Compression: ldbopts.NoCompression, NoSync: false, Strict: ldbopts.StrictAll, ReadOnly: false, }) + if err != nil { + return xerrors.Errorf("opening backup datastore at %s: %w", backupDir, err) + } // Export the key / values prefixes := []string{ @@ -197,7 +194,7 @@ var marketExportDatastoreCmd = &cli.Command{ fpath := path.Join(backupDir, "markets.datastore.backup") out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - return xerrors.Errorf("open %s: %w", fpath, err) + return xerrors.Errorf("opening backup file %s: %w", fpath, err) } // Write the backup to the file @@ -236,3 +233,77 @@ func exportPrefix(prefix string, ds datastore.Batching, backupDs datastore.Batch return nil } + +var marketImportDatastoreCmd = &cli.Command{ + Name: "import-datastore", + Description: "import markets datastore key/values from a backup file", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Usage: "path to the repo", + }, + &cli.StringFlag{ + Name: "backup-path", + Usage: "path to the backup directory", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + logging.SetLogLevel("badger", "ERROR") // nolint:errcheck + + backupPath := cctx.String("backup-path") + + // Open the repo at the repo path + lr, err := openLockedRepo(cctx.String("repo")) + if err != nil { + return err + } + defer lr.Close() //nolint:errcheck + + // Open the metadata datastore on the repo + repoDs, err := lr.Datastore(cctx.Context, datastore.NewKey(mktsMetadataNamespace).String()) + if err != nil { + return err + } + + r, err := os.Open(backupPath) + if err != nil { + return xerrors.Errorf("opening backup path %s: %w", backupPath, err) + } + + fmt.Println("Importing from backup file " + backupPath) + err = backupds.RestoreInto(r, repoDs) + if err != nil { + return xerrors.Errorf("restoring backup from path %s: %w", backupPath, err) + } + + fmt.Println("Completed importing from backup file " + backupPath) + + return nil + }, +} + +func openLockedRepo(path string) (repo.LockedRepo, error) { + // Open the repo at the repo path + rpo, err := repo.NewFS(path) + if err != nil { + return nil, xerrors.Errorf("could not open repo %s: %w", path, err) + } + + // Make sure the repo exists + exists, err := rpo.Exists() + if err != nil { + return nil, xerrors.Errorf("checking repo %s exists: %w", path, err) + } + if !exists { + return nil, xerrors.Errorf("repo does not exist: %s", path) + } + + // Lock the repo + lr, err := rpo.Lock(repo.StorageMiner) + if err != nil { + return nil, xerrors.Errorf("locking repo %s: %w", path, err) + } + + return lr, nil +} From 9d61f91cdf8a431f167ddbfffeee22c6ebf9cb4e Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 26 Jul 2021 16:41:22 +0200 Subject: [PATCH 4/4] Update cmd/lotus-shed/market.go Co-authored-by: Aayush Rajasekaran --- cmd/lotus-shed/market.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/market.go b/cmd/lotus-shed/market.go index eb1b56a94..8221e53eb 100644 --- a/cmd/lotus-shed/market.go +++ b/cmd/lotus-shed/market.go @@ -244,7 +244,7 @@ var marketImportDatastoreCmd = &cli.Command{ }, &cli.StringFlag{ Name: "backup-path", - Usage: "path to the backup directory", + Usage: "path to the backup file", Required: true, }, },