From 898454ca9afd7ba8e2f173b417c28e37cb346898 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jul 2019 15:23:29 +0200 Subject: [PATCH 1/4] Introduce fsrepo License: MIT Signed-off-by: Jakub Sztandera --- go.mod | 2 + go.sum | 4 ++ node/repo/fsrepo.go | 155 +++++++++++++++++++++++++++++++++++++++++++ node/repo/memrepo.go | 10 ++- 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 node/repo/fsrepo.go diff --git a/go.mod b/go.mod index c6e65e25e..b471457b8 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/ipfs/go-blockservice v0.0.2 github.com/ipfs/go-cid v0.0.2 github.com/ipfs/go-datastore v0.0.5 + github.com/ipfs/go-ds-badger v0.0.2 + github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90 github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f github.com/ipfs/go-ipfs-blockstore v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.0.1 diff --git a/go.sum b/go.sum index 4f704b363..286bf799b 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.2 h1:7ToQt7QByBhOTuZF2USMv+PGlMcBC7FW7FdgQ4FCsoo= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90 h1:DCJMKyZn7nkhthNLPZUly0bWLa8kE1UGhmX/sH9fWYw= +github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f h1:CpQZA1HsuaRQaFIUq7h/KqSyclyp/LrpcyifPnKRT2k= github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f/go.mod h1:WrX60HHX2SeMb602Z1s9Ztnf/4fzNHzwH9gxNTVpEmk= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= @@ -449,6 +451,8 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go4.org v0.0.0-20190218023631-ce4c26f7be8e h1:m9LfARr2VIOW0vsV19kEKp/sWQvZnGobA8JHui/XJoY= +go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go new file mode 100644 index 000000000..88ad688a6 --- /dev/null +++ b/node/repo/fsrepo.go @@ -0,0 +1,155 @@ +package repo + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/ipfs/go-datastore" + badger "github.com/ipfs/go-ds-badger" + fslock "github.com/ipfs/go-fs-lock" + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/multiformats/go-multiaddr" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-lotus/node/config" +) + +const ( + fsAPI = "api" + fsConfig = "config.toml" + fsDatastore = "datastore" + fsLibp2pKey = "libp2p.priv" + fsLock = "repo.lock" +) + +type FsRepo struct { + path string +} + +var _ Repo = &FsRepo{} + +func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) { + p := filepath.Join(fsr.path, fsAPI) + f, err := os.Open(p) + + if os.IsNotExist(err) { + return nil, ErrNoAPIEndpoint + } else if err != nil { + return nil, err + } + defer f.Close() //nolint: errcheck // Read only op + + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + strma := string(data) + strma = strings.TrimSpace(strma) + + apima, err := multiaddr.NewMultiaddr(strma) + if err != nil { + return nil, err + } + return apima, nil +} + +func (fsr *FsRepo) Lock() (LockedRepo, error) { + closer, err := fslock.Lock(fsr.path, fsLock) + if err != nil { + return nil, xerrors.Errorf("could not lock the repo: %w", err) + } + return &fsLockedRepo{ + path: fsr.path, + closer: closer, + }, nil +} + +type fsLockedRepo struct { + path string + closer io.Closer +} + +func (fsr *fsLockedRepo) Close() error { + err := fsr.closer.Close() + fsr.closer = nil + return err +} + +// join joins path elements with fsr.path +func (fsr *fsLockedRepo) join(paths ...string) string { + return filepath.Join(append([]string{fsr.path}, paths...)...) +} + +func (fsr *fsLockedRepo) stillValid() error { + if fsr.closer == nil { + return ErrClosedRepo + } + return nil +} + +func (fsr *fsLockedRepo) Datastore() (datastore.Datastore, error) { + return badger.NewDatastore(fsr.join(fsDatastore), nil) +} + +func (fsr *fsLockedRepo) Config() (*config.Root, error) { + if err := fsr.stillValid(); err != nil { + return nil, err + } + return config.FromFile(fsr.join(fsConfig)) +} + +func (fsr *fsLockedRepo) Libp2pIdentity() (crypto.PrivKey, error) { + kpath := fsr.join(fsLibp2pKey) + stat, err := os.Stat(kpath) + + if os.IsNotExist(err) { + pk, err := genLibp2pKey() + if err != nil { + return nil, xerrors.Errorf("could not generate private key: %w", err) + } + pkb, err := pk.Bytes() + if err != nil { + return nil, xerrors.Errorf("could not serialize private key: %w", err) + } + err = ioutil.WriteFile(kpath, pkb, 0600) + if err != nil { + return nil, xerrors.Errorf("could not write private key: %w", err) + } + } + + if stat.Mode()&0066 != 0 { + return nil, xerrors.New("libp2p identity has too wide access permissions, " + + fsLibp2pKey + " should have permission 0600") + } + + f, err := os.Open(kpath) + if err != nil { + return nil, xerrors.Errorf("could not open private key file: %w", err) + } + defer f.Close() //nolint: errcheck // read-only op + + pkbytes, err := ioutil.ReadAll(f) + if err != nil { + return nil, xerrors.Errorf("could not read private key file: %w", err) + } + + pk, err := crypto.UnmarshalPrivateKey(pkbytes) + if err != nil { + return nil, xerrors.Errorf("could not unmarshal private key: %w", err) + } + return pk, nil +} + +func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { + if err := fsr.stillValid(); err != nil { + return err + } + return ioutil.WriteFile(fsr.join(fsAPI), ma.Bytes(), 0666) +} + +func (fsr *fsLockedRepo) Wallet() (interface{}, error) { + panic("not implemented") +} diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index de2ace933..cf48cc317 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -44,6 +44,14 @@ type MemRepoOptions struct { Wallet interface{} } +func genLibp2pKey() (crypto.PrivKey, error) { + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + if err != nil { + return nil, err + } + return pk, nil +} + // NewMemory creates new memory based repo with provided options. // opts can be nil, it will be replaced with defaults. // Any field in opts can be nil, they will be replaced by defaults. @@ -58,7 +66,7 @@ func NewMemory(opts *MemRepoOptions) *MemRepo { opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore()) } if opts.Libp2pKey == nil { - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + pk, err := genLibp2pKey() if err != nil { panic(err) } From 8cd484a8b031fd470fdff3df3fd3b605c64d42b7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jul 2019 15:35:00 +0200 Subject: [PATCH 2/4] Change lint options License: MIT Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 13 +++++++++---- .golangci.yml | 1 - node/repo/fsrepo.go | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb8fe945c..5f0e29313 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ jobs: command: | bash <(curl -s https://codecov.io/bash) - lint: + lint: &lint description: | Run golangci-lint. parameters: @@ -124,15 +124,20 @@ jobs: command: | golangci-lint run -v \ --concurrency << parameters.concurrency >> << parameters.args >> + lint-changes: + <<: *lint + + lint-all: + <<: *lint workflows: version: 2 ci: jobs: - - lint - - lint: - args: "--no-config --exclude-use-default=false --disable-all --enable golint" + - lint-changes: + args: "--new-from-rev origin/master" + - lint-all - test: codecov-upload: true - mod-tidy-check diff --git a/.golangci.yml b/.golangci.yml index f873557aa..8f25233e6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,7 +20,6 @@ issues: exclude: - "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this" - "Potential file inclusion via variable" - - "should have( a package)? comment" exclude-use-default: false exclude-rules: diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 88ad688a6..cfcb52f8d 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -25,12 +25,14 @@ const ( fsLock = "repo.lock" ) +// FsRepo is struct for repo, use NewFS to create type FsRepo struct { path string } var _ Repo = &FsRepo{} +// APIEndpoint returns endpoint of API in this repo func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) { p := filepath.Join(fsr.path, fsAPI) f, err := os.Open(p) @@ -56,6 +58,7 @@ func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) { return apima, nil } +// Lock acquires exclusive lock on this repo func (fsr *FsRepo) Lock() (LockedRepo, error) { closer, err := fslock.Lock(fsr.path, fsLock) if err != nil { From d484e4ffb907b1b254bc2d14eadbb46a75943da7 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jul 2019 17:10:45 +0200 Subject: [PATCH 3/4] Run tests and fix some bugs License: MIT Signed-off-by: Jakub Sztandera --- node/repo/fsrepo.go | 24 +++++++++++++++-- node/repo/fsrepo_test.go | 26 ++++++++++++++++++ node/repo/memrepo_test.go | 51 ++---------------------------------- node/repo/repo_test.go | 55 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 51 deletions(-) create mode 100644 node/repo/fsrepo_test.go create mode 100644 node/repo/repo_test.go diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index cfcb52f8d..0e4ad472c 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -32,6 +32,12 @@ type FsRepo struct { var _ Repo = &FsRepo{} +func NewFS(path string) (*FsRepo, error) { + return &FsRepo{ + path: path, + }, nil +} + // APIEndpoint returns endpoint of API in this repo func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) { p := filepath.Join(fsr.path, fsAPI) @@ -60,6 +66,14 @@ func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) { // Lock acquires exclusive lock on this repo func (fsr *FsRepo) Lock() (LockedRepo, error) { + locked, err := fslock.Locked(fsr.path, fsLock) + if err != nil { + return nil, xerrors.Errorf("could not check lock status: %w", err) + } + if locked { + return nil, ErrRepoAlreadyLocked + } + closer, err := fslock.Lock(fsr.path, fsLock) if err != nil { return nil, xerrors.Errorf("could not lock the repo: %w", err) @@ -76,7 +90,13 @@ type fsLockedRepo struct { } func (fsr *fsLockedRepo) Close() error { - err := fsr.closer.Close() + err := os.Remove(fsr.join(fsAPI)) + + if err != nil && !os.IsNotExist(err) { + return xerrors.Errorf("could not remove API file: %w", err) + } + + err = fsr.closer.Close() fsr.closer = nil return err } @@ -150,7 +170,7 @@ func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error { if err := fsr.stillValid(); err != nil { return err } - return ioutil.WriteFile(fsr.join(fsAPI), ma.Bytes(), 0666) + return ioutil.WriteFile(fsr.join(fsAPI), []byte(ma.String()), 0666) } func (fsr *fsLockedRepo) Wallet() (interface{}, error) { diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go new file mode 100644 index 000000000..0275debdd --- /dev/null +++ b/node/repo/fsrepo_test.go @@ -0,0 +1,26 @@ +package repo + +import "io/ioutil" +import "os" +import "testing" + +func genFsRepo(t *testing.T) (*FsRepo, func()) { + path, err := ioutil.TempDir("", "lotus-repo-*") + if err != nil { + t.Fatal(err) + } + + repo, err := NewFS(path) + if err != nil { + t.Fatal(err) + } + return repo, func() { + os.RemoveAll(path) + } +} + +func TestFsBasic(t *testing.T) { + repo, closer := genFsRepo(t) + defer closer() + basicTest(t, repo) +} diff --git a/node/repo/memrepo_test.go b/node/repo/memrepo_test.go index 27b6d9f4b..965bc02c1 100644 --- a/node/repo/memrepo_test.go +++ b/node/repo/memrepo_test.go @@ -2,56 +2,9 @@ package repo import ( "testing" - - "github.com/multiformats/go-multiaddr" - "github.com/stretchr/testify/assert" ) -func TestMemRepo(t *testing.T) { +func TestMemBasic(t *testing.T) { repo := NewMemory(nil) - apima, err := repo.APIEndpoint() - if assert.Error(t, err) { - assert.Equal(t, ErrNoAPIEndpoint, err) - } - assert.Nil(t, apima, "with no api endpoint, return should be nil") - - lrepo, err := repo.Lock() - assert.NoError(t, err, "should be able to lock once") - assert.NotNil(t, lrepo, "locked repo shouldn't be nil") - - { - lrepo2, err := repo.Lock() - if assert.Error(t, err) { - assert.Equal(t, ErrRepoAlreadyLocked, err) - } - assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned") - } - - err = lrepo.Close() - assert.NoError(t, err, "should be able to unlock") - - lrepo, err = repo.Lock() - assert.NoError(t, err, "should be able to relock") - assert.NotNil(t, lrepo, "locked repo shouldn't be nil") - - ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244") - assert.NoError(t, err, "creating multiaddr shouldn't error") - - err = lrepo.SetAPIEndpoint(ma) - assert.NoError(t, err, "setting multiaddr shouldn't error") - - apima, err = repo.APIEndpoint() - assert.NoError(t, err, "setting multiaddr shouldn't error") - assert.Equal(t, ma, apima, "returned API multiaddr should be the same") - - err = lrepo.Close() - assert.NoError(t, err, "should be able to close") - - apima, err = repo.APIEndpoint() - - if assert.Error(t, err) { - assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil") - } - assert.Nil(t, apima, "with closed repo, apima should be set back to nil") - + basicTest(t, repo) } diff --git a/node/repo/repo_test.go b/node/repo/repo_test.go new file mode 100644 index 000000000..2e90534a8 --- /dev/null +++ b/node/repo/repo_test.go @@ -0,0 +1,55 @@ +package repo + +import ( + "testing" + + "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/assert" +) + +func basicTest(t *testing.T, repo Repo) { + apima, err := repo.APIEndpoint() + if assert.Error(t, err) { + assert.Equal(t, ErrNoAPIEndpoint, err) + } + assert.Nil(t, apima, "with no api endpoint, return should be nil") + + lrepo, err := repo.Lock() + assert.NoError(t, err, "should be able to lock once") + assert.NotNil(t, lrepo, "locked repo shouldn't be nil") + + { + lrepo2, err := repo.Lock() + if assert.Error(t, err) { + assert.Equal(t, ErrRepoAlreadyLocked, err) + } + assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned") + } + + err = lrepo.Close() + assert.NoError(t, err, "should be able to unlock") + + lrepo, err = repo.Lock() + assert.NoError(t, err, "should be able to relock") + assert.NotNil(t, lrepo, "locked repo shouldn't be nil") + + ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244") + assert.NoError(t, err, "creating multiaddr shouldn't error") + + err = lrepo.SetAPIEndpoint(ma) + assert.NoError(t, err, "setting multiaddr shouldn't error") + + apima, err = repo.APIEndpoint() + assert.NoError(t, err, "setting multiaddr shouldn't error") + assert.Equal(t, ma, apima, "returned API multiaddr should be the same") + + err = lrepo.Close() + assert.NoError(t, err, "should be able to close") + + apima, err = repo.APIEndpoint() + + if assert.Error(t, err) { + assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil") + } + assert.Nil(t, apima, "with closed repo, apima should be set back to nil") +} From ed54a30109c0ce550fcf47de010f7b0f01d094a9 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 10 Jul 2019 17:14:29 +0200 Subject: [PATCH 4/4] fix lint License: MIT Signed-off-by: Jakub Sztandera --- node/repo/fsrepo.go | 1 + node/repo/fsrepo_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 0e4ad472c..1d3f313ee 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -32,6 +32,7 @@ type FsRepo struct { var _ Repo = &FsRepo{} +// NewFS creates a repo instance based on a path on file system func NewFS(path string) (*FsRepo, error) { return &FsRepo{ path: path, diff --git a/node/repo/fsrepo_test.go b/node/repo/fsrepo_test.go index 0275debdd..4cda09f4e 100644 --- a/node/repo/fsrepo_test.go +++ b/node/repo/fsrepo_test.go @@ -15,7 +15,7 @@ func genFsRepo(t *testing.T) (*FsRepo, func()) { t.Fatal(err) } return repo, func() { - os.RemoveAll(path) + _ = os.RemoveAll(path) } }