Merge pull request #15 from filecoin-project/feat/fs-repo

Introduce fsrepo

.circleci/config.yml

@@ -89,7 +89,7 @@ jobs:
                 command: |
                   bash <(curl -s https://codecov.io/bash)
 
-  lint:
+  lint: &lint
     description: |
       Run golangci-lint.
     parameters:
@@ -124,15 +124,20 @@ jobs:
           command: |
             golangci-lint run -v \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
+  lint-all:
+    <<: *lint
 
 
 workflows:
   version: 2
   ci:
     jobs:
-      - lint
+      - lint-changes:
-      - lint:
+          args: "--new-from-rev origin/master"
-          args: "--no-config --exclude-use-default=false --disable-all --enable golint"
+      - lint-all
       - test:
           codecov-upload: true
       - mod-tidy-check

.golangci.yml

@@ -20,7 +20,6 @@ issues:
   exclude:
     - "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this"
     - "Potential file inclusion via variable"
-    - "should have( a package)? comment"
 
   exclude-use-default: false
   exclude-rules:

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/ipfs/go-blockservice v0.0.2
 	github.com/ipfs/go-cid v0.0.2
 	github.com/ipfs/go-datastore v0.0.5
+	github.com/ipfs/go-ds-badger v0.0.2
+	github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90
 	github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f
 	github.com/ipfs/go-ipfs-blockstore v0.0.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1

go.sum

@@ -107,6 +107,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U
 github.com/ipfs/go-ds-badger v0.0.2 h1:7ToQt7QByBhOTuZF2USMv+PGlMcBC7FW7FdgQ4FCsoo=
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
+github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90 h1:DCJMKyZn7nkhthNLPZUly0bWLa8kE1UGhmX/sH9fWYw=
+github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
 github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f h1:CpQZA1HsuaRQaFIUq7h/KqSyclyp/LrpcyifPnKRT2k=
 github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f/go.mod h1:WrX60HHX2SeMb602Z1s9Ztnf/4fzNHzwH9gxNTVpEmk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc=
@@ -449,6 +451,8 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go4.org v0.0.0-20190218023631-ce4c26f7be8e h1:m9LfARr2VIOW0vsV19kEKp/sWQvZnGobA8JHui/XJoY=
+go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

node/repo/fsrepo.go

@@ -0,0 +1,179 @@
+package repo
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ipfs/go-datastore"
+	badger "github.com/ipfs/go-ds-badger"
+	fslock "github.com/ipfs/go-fs-lock"
+	"github.com/libp2p/go-libp2p-core/crypto"
+	"github.com/multiformats/go-multiaddr"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-lotus/node/config"
+)
+
+const (
+	fsAPI       = "api"
+	fsConfig    = "config.toml"
+	fsDatastore = "datastore"
+	fsLibp2pKey = "libp2p.priv"
+	fsLock      = "repo.lock"
+)
+
+// FsRepo is struct for repo, use NewFS to create
+type FsRepo struct {
+	path string
+}
+
+var _ Repo = &FsRepo{}
+
+// NewFS creates a repo instance based on a path on file system
+func NewFS(path string) (*FsRepo, error) {
+	return &FsRepo{
+		path: path,
+	}, nil
+}
+
+// APIEndpoint returns endpoint of API in this repo
+func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) {
+	p := filepath.Join(fsr.path, fsAPI)
+	f, err := os.Open(p)
+
+	if os.IsNotExist(err) {
+		return nil, ErrNoAPIEndpoint
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close() //nolint: errcheck // Read only op
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	strma := string(data)
+	strma = strings.TrimSpace(strma)
+
+	apima, err := multiaddr.NewMultiaddr(strma)
+	if err != nil {
+		return nil, err
+	}
+	return apima, nil
+}
+
+// Lock acquires exclusive lock on this repo
+func (fsr *FsRepo) Lock() (LockedRepo, error) {
+	locked, err := fslock.Locked(fsr.path, fsLock)
+	if err != nil {
+		return nil, xerrors.Errorf("could not check lock status: %w", err)
+	}
+	if locked {
+		return nil, ErrRepoAlreadyLocked
+	}
+
+	closer, err := fslock.Lock(fsr.path, fsLock)
+	if err != nil {
+		return nil, xerrors.Errorf("could not lock the repo: %w", err)
+	}
+	return &fsLockedRepo{
+		path:   fsr.path,
+		closer: closer,
+	}, nil
+}
+
+type fsLockedRepo struct {
+	path   string
+	closer io.Closer
+}
+
+func (fsr *fsLockedRepo) Close() error {
+	err := os.Remove(fsr.join(fsAPI))
+
+	if err != nil && !os.IsNotExist(err) {
+		return xerrors.Errorf("could not remove API file: %w", err)
+	}
+
+	err = fsr.closer.Close()
+	fsr.closer = nil
+	return err
+}
+
+// join joins path elements with fsr.path
+func (fsr *fsLockedRepo) join(paths ...string) string {
+	return filepath.Join(append([]string{fsr.path}, paths...)...)
+}
+
+func (fsr *fsLockedRepo) stillValid() error {
+	if fsr.closer == nil {
+		return ErrClosedRepo
+	}
+	return nil
+}
+
+func (fsr *fsLockedRepo) Datastore() (datastore.Datastore, error) {
+	return badger.NewDatastore(fsr.join(fsDatastore), nil)
+}
+
+func (fsr *fsLockedRepo) Config() (*config.Root, error) {
+	if err := fsr.stillValid(); err != nil {
+		return nil, err
+	}
+	return config.FromFile(fsr.join(fsConfig))
+}
+
+func (fsr *fsLockedRepo) Libp2pIdentity() (crypto.PrivKey, error) {
+	kpath := fsr.join(fsLibp2pKey)
+	stat, err := os.Stat(kpath)
+
+	if os.IsNotExist(err) {
+		pk, err := genLibp2pKey()
+		if err != nil {
+			return nil, xerrors.Errorf("could not generate private key: %w", err)
+		}
+		pkb, err := pk.Bytes()
+		if err != nil {
+			return nil, xerrors.Errorf("could not serialize private key: %w", err)
+		}
+		err = ioutil.WriteFile(kpath, pkb, 0600)
+		if err != nil {
+			return nil, xerrors.Errorf("could not write private key: %w", err)
+		}
+	}
+
+	if stat.Mode()&0066 != 0 {
+		return nil, xerrors.New("libp2p identity has too wide access permissions, " +
+			fsLibp2pKey + " should have permission 0600")
+	}
+
+	f, err := os.Open(kpath)
+	if err != nil {
+		return nil, xerrors.Errorf("could not open private key file: %w", err)
+	}
+	defer f.Close() //nolint: errcheck // read-only op
+
+	pkbytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, xerrors.Errorf("could not read private key file: %w", err)
+	}
+
+	pk, err := crypto.UnmarshalPrivateKey(pkbytes)
+	if err != nil {
+		return nil, xerrors.Errorf("could not unmarshal private key: %w", err)
+	}
+	return pk, nil
+}
+
+func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error {
+	if err := fsr.stillValid(); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(fsr.join(fsAPI), []byte(ma.String()), 0666)
+}
+
+func (fsr *fsLockedRepo) Wallet() (interface{}, error) {
+	panic("not implemented")
+}

node/repo/fsrepo_test.go

@@ -0,0 +1,26 @@
+package repo
+
+import "io/ioutil"
+import "os"
+import "testing"
+
+func genFsRepo(t *testing.T) (*FsRepo, func()) {
+	path, err := ioutil.TempDir("", "lotus-repo-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	repo, err := NewFS(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return repo, func() {
+		_ = os.RemoveAll(path)
+	}
+}
+
+func TestFsBasic(t *testing.T) {
+	repo, closer := genFsRepo(t)
+	defer closer()
+	basicTest(t, repo)
+}

node/repo/memrepo.go

@@ -44,6 +44,14 @@ type MemRepoOptions struct {
 	Wallet    interface{}
 }
 
+func genLibp2pKey() (crypto.PrivKey, error) {
+	pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	return pk, nil
+}
+
 // NewMemory creates new memory based repo with provided options.
 // opts can be nil, it  will be replaced with defaults.
 // Any field in opts can be nil, they will be replaced by defaults.
@@ -58,7 +66,7 @@ func NewMemory(opts *MemRepoOptions) *MemRepo {
 		opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore())
 	}
 	if opts.Libp2pKey == nil {
-		pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+		pk, err := genLibp2pKey()
 		if err != nil {
 			panic(err)
 		}

node/repo/memrepo_test.go

@@ -2,56 +2,9 @@ package repo
 
 import (
 	"testing"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/stretchr/testify/assert"
 )
 
-func TestMemRepo(t *testing.T) {
+func TestMemBasic(t *testing.T) {
 	repo := NewMemory(nil)
-	apima, err := repo.APIEndpoint()
+	basicTest(t, repo)
-	if assert.Error(t, err) {
-		assert.Equal(t, ErrNoAPIEndpoint, err)
-	}
-	assert.Nil(t, apima, "with no api endpoint, return should be nil")
-
-	lrepo, err := repo.Lock()
-	assert.NoError(t, err, "should be able to lock once")
-	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
-
-	{
-		lrepo2, err := repo.Lock()
-		if assert.Error(t, err) {
-			assert.Equal(t, ErrRepoAlreadyLocked, err)
-		}
-		assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned")
-	}
-
-	err = lrepo.Close()
-	assert.NoError(t, err, "should be able to unlock")
-
-	lrepo, err = repo.Lock()
-	assert.NoError(t, err, "should be able to relock")
-	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
-
-	ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244")
-	assert.NoError(t, err, "creating multiaddr shouldn't error")
-
-	err = lrepo.SetAPIEndpoint(ma)
-	assert.NoError(t, err, "setting multiaddr shouldn't error")
-
-	apima, err = repo.APIEndpoint()
-	assert.NoError(t, err, "setting multiaddr shouldn't error")
-	assert.Equal(t, ma, apima, "returned API multiaddr should be the same")
-
-	err = lrepo.Close()
-	assert.NoError(t, err, "should be able to close")
-
-	apima, err = repo.APIEndpoint()
-
-	if assert.Error(t, err) {
-		assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil")
-	}
-	assert.Nil(t, apima, "with closed repo, apima should be set back to nil")
-
 }

node/repo/repo_test.go

@@ -0,0 +1,55 @@
+package repo
+
+import (
+	"testing"
+
+	"github.com/multiformats/go-multiaddr"
+	"github.com/stretchr/testify/assert"
+)
+
+func basicTest(t *testing.T, repo Repo) {
+	apima, err := repo.APIEndpoint()
+	if assert.Error(t, err) {
+		assert.Equal(t, ErrNoAPIEndpoint, err)
+	}
+	assert.Nil(t, apima, "with no api endpoint, return should be nil")
+
+	lrepo, err := repo.Lock()
+	assert.NoError(t, err, "should be able to lock once")
+	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
+
+	{
+		lrepo2, err := repo.Lock()
+		if assert.Error(t, err) {
+			assert.Equal(t, ErrRepoAlreadyLocked, err)
+		}
+		assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned")
+	}
+
+	err = lrepo.Close()
+	assert.NoError(t, err, "should be able to unlock")
+
+	lrepo, err = repo.Lock()
+	assert.NoError(t, err, "should be able to relock")
+	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
+
+	ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244")
+	assert.NoError(t, err, "creating multiaddr shouldn't error")
+
+	err = lrepo.SetAPIEndpoint(ma)
+	assert.NoError(t, err, "setting multiaddr shouldn't error")
+
+	apima, err = repo.APIEndpoint()
+	assert.NoError(t, err, "setting multiaddr shouldn't error")
+	assert.Equal(t, ma, apima, "returned API multiaddr should be the same")
+
+	err = lrepo.Close()
+	assert.NoError(t, err, "should be able to close")
+
+	apima, err = repo.APIEndpoint()
+
+	if assert.Error(t, err) {
+		assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil")
+	}
+	assert.Nil(t, apima, "with closed repo, apima should be set back to nil")
+}