Merge remote-tracking branch 'origin/master' into feat/repo-daemon

.circleci/config.yml

@@ -89,7 +89,7 @@ jobs:
                 command: |
                   bash <(curl -s https://codecov.io/bash)
 
-  lint:
+  lint: &lint
     description: |
       Run golangci-lint.
     parameters:
@@ -124,15 +124,20 @@ jobs:
           command: |
             golangci-lint run -v \
               --concurrency << parameters.concurrency >> << parameters.args >>
+  lint-changes:
+    <<: *lint
+
+  lint-all:
+    <<: *lint
 
 
 workflows:
   version: 2
   ci:
     jobs:
-      - lint
-      - lint:
-          args: "--no-config --exclude-use-default=false --disable-all --enable golint"
+      - lint-changes:
+          args: "--new-from-rev origin/master"
+      - lint-all
       - test:
           codecov-upload: true
       - mod-tidy-check

.golangci.yml

@@ -20,7 +20,6 @@ issues:
   exclude:
     - "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this"
     - "Potential file inclusion via variable"
-    - "should have( a package)? comment"
 
   exclude-use-default: false
   exclude-rules:

chain/types.go

@@ -425,6 +425,10 @@ func (ts *TipSet) Height() uint64 {
 	return ts.height
 }
 
+func (ts *TipSet) Weight() uint64 {
+	panic("if tipsets are going to have weight on them, we need to wire that through")
+}
+
 func (ts *TipSet) Parents() []cid.Cid {
 	return ts.blks[0].Parents
 }

go.mod

@@ -10,6 +10,8 @@ require (
 	github.com/ipfs/go-blockservice v0.0.2
 	github.com/ipfs/go-cid v0.0.2
 	github.com/ipfs/go-datastore v0.0.5
+	github.com/ipfs/go-ds-badger v0.0.2
+	github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90
 	github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f
 	github.com/ipfs/go-ipfs-blockstore v0.0.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1

go.sum

@@ -107,6 +107,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U
 github.com/ipfs/go-ds-badger v0.0.2 h1:7ToQt7QByBhOTuZF2USMv+PGlMcBC7FW7FdgQ4FCsoo=
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
+github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90 h1:DCJMKyZn7nkhthNLPZUly0bWLa8kE1UGhmX/sH9fWYw=
+github.com/ipfs/go-fs-lock v0.0.0-20190710120703-5ab4a142bd90/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
 github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f h1:CpQZA1HsuaRQaFIUq7h/KqSyclyp/LrpcyifPnKRT2k=
 github.com/ipfs/go-hamt-ipld v0.0.0-20190613164304-cd074602062f/go.mod h1:WrX60HHX2SeMb602Z1s9Ztnf/4fzNHzwH9gxNTVpEmk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc=
@@ -449,6 +451,8 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go4.org v0.0.0-20190218023631-ce4c26f7be8e h1:m9LfARr2VIOW0vsV19kEKp/sWQvZnGobA8JHui/XJoY=
+go4.org v0.0.0-20190218023631-ce4c26f7be8e/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

miner/miner.go

@@ -0,0 +1,161 @@
+package miner
+
+import (
+	"context"
+	"time"
+
+	logging "github.com/ipfs/go-log"
+	"github.com/pkg/errors"
+
+	chain "github.com/filecoin-project/go-lotus/chain"
+)
+
+var log = logging.Logger("miner")
+
+type api interface {
+	SubmitNewBlock(blk *chain.BlockMsg) error
+
+	// returns a set of messages that havent been included in the chain as of
+	// the given tipset
+	PendingMessages(base *chain.TipSet) ([]*chain.SignedMessage, error)
+
+	// Returns the best tipset for the miner to mine on top of.
+	// TODO: Not sure this feels right (including the messages api). Miners
+	// will likely want to have more control over exactly which blocks get
+	// mined on, and which messages are included.
+	GetBestTipset() (*chain.TipSet, error)
+
+	// returns the lookback randomness from the chain used for the election
+	GetChainRandomness(ts *chain.TipSet) ([]byte, error)
+
+	// create a block
+	// it seems realllllly annoying to do all the actions necessary to build a
+	// block through the API. so, we just add the block creation to the API
+	// now, all the 'miner' does is check if they win, and call create block
+	CreateBlock(base *chain.TipSet, tickets []chain.Ticket, eproof chain.ElectionProof, msgs []*chain.SignedMessage) (*chain.BlockMsg, error)
+}
+
+type Miner struct {
+	api api
+
+	// time between blocks, network parameter
+	Delay time.Duration
+
+	lastWork *MiningBase
+}
+
+func (m *Miner) Mine(ctx context.Context) {
+	for {
+		base, err := m.GetBestMiningCandidate()
+		if err != nil {
+			log.Errorf("failed to get best mining candidate: %s", err)
+			continue
+		}
+
+		b, err := m.mineOne(ctx, base)
+		if err != nil {
+			log.Errorf("mining block failed: %s", err)
+			continue
+		}
+
+		if b != nil {
+			if err := m.api.SubmitNewBlock(b); err != nil {
+				log.Errorf("failed to submit newly mined block: %s", err)
+			}
+		}
+	}
+}
+
+type MiningBase struct {
+	ts      *chain.TipSet
+	tickets []chain.Ticket
+}
+
+func (m *Miner) GetBestMiningCandidate() (*MiningBase, error) {
+	bts, err := m.api.GetBestTipset()
+	if err != nil {
+		return nil, err
+	}
+
+	if m.lastWork != nil {
+		if m.lastWork.ts.Equals(bts) {
+			return m.lastWork, nil
+		}
+
+		if bts.Weight() <= m.lastWork.ts.Weight() {
+			return m.lastWork, nil
+		}
+	}
+
+	return &MiningBase{
+		ts: bts,
+	}, nil
+}
+
+func (m *Miner) mineOne(ctx context.Context, base *MiningBase) (*chain.BlockMsg, error) {
+	log.Info("mine one")
+	ticket, err := m.scratchTicket(ctx, base)
+	if err != nil {
+		return nil, errors.Wrap(err, "scratching ticket failed")
+	}
+
+	win, proof, err := m.isWinnerNextRound(base)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to check if we win next round")
+	}
+
+	if !win {
+		m.submitNullTicket(base, ticket)
+		return nil, nil
+	}
+
+	b, err := m.createBlock(base, ticket, proof)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create block")
+	}
+	log.Infof("created new block: %s", b.Cid())
+
+	return b, nil
+}
+
+func (m *Miner) submitNullTicket(base *MiningBase, ticket chain.Ticket) {
+	base.tickets = append(base.tickets, ticket)
+	m.lastWork = base
+}
+
+func (m *Miner) isWinnerNextRound(base *MiningBase) (bool, chain.ElectionProof, error) {
+	r, err := m.api.GetChainRandomness(base.ts)
+	if err != nil {
+		return false, nil, err
+	}
+
+	_ = r // TODO: use this to properly compute the election proof
+
+	return true, []byte("election prooooof"), nil
+}
+
+func (m *Miner) scratchTicket(ctx context.Context, base *MiningBase) (chain.Ticket, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-time.After(m.Delay):
+	}
+
+	return []byte("this is a ticket"), nil
+}
+
+func (m *Miner) createBlock(base *MiningBase, ticket chain.Ticket, proof chain.ElectionProof) (*chain.BlockMsg, error) {
+
+	pending, err := m.api.PendingMessages(base.ts)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get pending messages")
+	}
+
+	// why even return this? that api call could just submit it for us
+	return m.api.CreateBlock(base.ts, append(base.tickets, ticket), proof, pending)
+}
+
+func (m *Miner) selectMessages(msgs []*chain.SignedMessage) []*chain.SignedMessage {
+	// TODO: filter and select 'best' message if too many to fit in one block
+	return msgs
+}

node/repo/fsrepo.go

@@ -0,0 +1,179 @@
+package repo
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ipfs/go-datastore"
+	badger "github.com/ipfs/go-ds-badger"
+	fslock "github.com/ipfs/go-fs-lock"
+	"github.com/libp2p/go-libp2p-core/crypto"
+	"github.com/multiformats/go-multiaddr"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-lotus/node/config"
+)
+
+const (
+	fsAPI       = "api"
+	fsConfig    = "config.toml"
+	fsDatastore = "datastore"
+	fsLibp2pKey = "libp2p.priv"
+	fsLock      = "repo.lock"
+)
+
+// FsRepo is struct for repo, use NewFS to create
+type FsRepo struct {
+	path string
+}
+
+var _ Repo = &FsRepo{}
+
+// NewFS creates a repo instance based on a path on file system
+func NewFS(path string) (*FsRepo, error) {
+	return &FsRepo{
+		path: path,
+	}, nil
+}
+
+// APIEndpoint returns endpoint of API in this repo
+func (fsr *FsRepo) APIEndpoint() (multiaddr.Multiaddr, error) {
+	p := filepath.Join(fsr.path, fsAPI)
+	f, err := os.Open(p)
+
+	if os.IsNotExist(err) {
+		return nil, ErrNoAPIEndpoint
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close() //nolint: errcheck // Read only op
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	strma := string(data)
+	strma = strings.TrimSpace(strma)
+
+	apima, err := multiaddr.NewMultiaddr(strma)
+	if err != nil {
+		return nil, err
+	}
+	return apima, nil
+}
+
+// Lock acquires exclusive lock on this repo
+func (fsr *FsRepo) Lock() (LockedRepo, error) {
+	locked, err := fslock.Locked(fsr.path, fsLock)
+	if err != nil {
+		return nil, xerrors.Errorf("could not check lock status: %w", err)
+	}
+	if locked {
+		return nil, ErrRepoAlreadyLocked
+	}
+
+	closer, err := fslock.Lock(fsr.path, fsLock)
+	if err != nil {
+		return nil, xerrors.Errorf("could not lock the repo: %w", err)
+	}
+	return &fsLockedRepo{
+		path:   fsr.path,
+		closer: closer,
+	}, nil
+}
+
+type fsLockedRepo struct {
+	path   string
+	closer io.Closer
+}
+
+func (fsr *fsLockedRepo) Close() error {
+	err := os.Remove(fsr.join(fsAPI))
+
+	if err != nil && !os.IsNotExist(err) {
+		return xerrors.Errorf("could not remove API file: %w", err)
+	}
+
+	err = fsr.closer.Close()
+	fsr.closer = nil
+	return err
+}
+
+// join joins path elements with fsr.path
+func (fsr *fsLockedRepo) join(paths ...string) string {
+	return filepath.Join(append([]string{fsr.path}, paths...)...)
+}
+
+func (fsr *fsLockedRepo) stillValid() error {
+	if fsr.closer == nil {
+		return ErrClosedRepo
+	}
+	return nil
+}
+
+func (fsr *fsLockedRepo) Datastore() (datastore.Datastore, error) {
+	return badger.NewDatastore(fsr.join(fsDatastore), nil)
+}
+
+func (fsr *fsLockedRepo) Config() (*config.Root, error) {
+	if err := fsr.stillValid(); err != nil {
+		return nil, err
+	}
+	return config.FromFile(fsr.join(fsConfig))
+}
+
+func (fsr *fsLockedRepo) Libp2pIdentity() (crypto.PrivKey, error) {
+	kpath := fsr.join(fsLibp2pKey)
+	stat, err := os.Stat(kpath)
+
+	if os.IsNotExist(err) {
+		pk, err := genLibp2pKey()
+		if err != nil {
+			return nil, xerrors.Errorf("could not generate private key: %w", err)
+		}
+		pkb, err := pk.Bytes()
+		if err != nil {
+			return nil, xerrors.Errorf("could not serialize private key: %w", err)
+		}
+		err = ioutil.WriteFile(kpath, pkb, 0600)
+		if err != nil {
+			return nil, xerrors.Errorf("could not write private key: %w", err)
+		}
+	}
+
+	if stat.Mode()&0066 != 0 {
+		return nil, xerrors.New("libp2p identity has too wide access permissions, " +
+			fsLibp2pKey + " should have permission 0600")
+	}
+
+	f, err := os.Open(kpath)
+	if err != nil {
+		return nil, xerrors.Errorf("could not open private key file: %w", err)
+	}
+	defer f.Close() //nolint: errcheck // read-only op
+
+	pkbytes, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, xerrors.Errorf("could not read private key file: %w", err)
+	}
+
+	pk, err := crypto.UnmarshalPrivateKey(pkbytes)
+	if err != nil {
+		return nil, xerrors.Errorf("could not unmarshal private key: %w", err)
+	}
+	return pk, nil
+}
+
+func (fsr *fsLockedRepo) SetAPIEndpoint(ma multiaddr.Multiaddr) error {
+	if err := fsr.stillValid(); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(fsr.join(fsAPI), []byte(ma.String()), 0666)
+}
+
+func (fsr *fsLockedRepo) Wallet() (interface{}, error) {
+	panic("not implemented")
+}

node/repo/fsrepo_test.go

@@ -0,0 +1,26 @@
+package repo
+
+import "io/ioutil"
+import "os"
+import "testing"
+
+func genFsRepo(t *testing.T) (*FsRepo, func()) {
+	path, err := ioutil.TempDir("", "lotus-repo-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	repo, err := NewFS(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return repo, func() {
+		_ = os.RemoveAll(path)
+	}
+}
+
+func TestFsBasic(t *testing.T) {
+	repo, closer := genFsRepo(t)
+	defer closer()
+	basicTest(t, repo)
+}

node/repo/memrepo.go

@@ -45,6 +45,14 @@ type MemRepoOptions struct {
 	Wallet    interface{}
 }
 
+func genLibp2pKey() (crypto.PrivKey, error) {
+	pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	return pk, nil
+}
+
 // NewMemory creates new memory based repo with provided options.
 // opts can be nil, it  will be replaced with defaults.
 // Any field in opts can be nil, they will be replaced by defaults.
@@ -59,7 +67,7 @@ func NewMemory(opts *MemRepoOptions) *MemRepo {
 		opts.Ds = dssync.MutexWrap(datastore.NewMapDatastore())
 	}
 	if opts.Libp2pKey == nil {
-		pk, _, err := crypto.GenerateEd25519Key(rand.Reader)
+		pk, err := genLibp2pKey()
 		if err != nil {
 			panic(err)
 		}

node/repo/memrepo_test.go

@@ -2,56 +2,9 @@ package repo
 
 import (
 	"testing"
-
-	"github.com/multiformats/go-multiaddr"
-	"github.com/stretchr/testify/assert"
 )
 
-func TestMemRepo(t *testing.T) {
+func TestMemBasic(t *testing.T) {
 	repo := NewMemory(nil)
-	apima, err := repo.APIEndpoint()
-	if assert.Error(t, err) {
-		assert.Equal(t, ErrNoAPIEndpoint, err)
-	}
-	assert.Nil(t, apima, "with no api endpoint, return should be nil")
-
-	lrepo, err := repo.Lock()
-	assert.NoError(t, err, "should be able to lock once")
-	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
-
-	{
-		lrepo2, err := repo.Lock()
-		if assert.Error(t, err) {
-			assert.Equal(t, ErrRepoAlreadyLocked, err)
-		}
-		assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned")
-	}
-
-	err = lrepo.Close()
-	assert.NoError(t, err, "should be able to unlock")
-
-	lrepo, err = repo.Lock()
-	assert.NoError(t, err, "should be able to relock")
-	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
-
-	ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244")
-	assert.NoError(t, err, "creating multiaddr shouldn't error")
-
-	err = lrepo.SetAPIEndpoint(ma)
-	assert.NoError(t, err, "setting multiaddr shouldn't error")
-
-	apima, err = repo.APIEndpoint()
-	assert.NoError(t, err, "setting multiaddr shouldn't error")
-	assert.Equal(t, ma, apima, "returned API multiaddr should be the same")
-
-	err = lrepo.Close()
-	assert.NoError(t, err, "should be able to close")
-
-	apima, err = repo.APIEndpoint()
-
-	if assert.Error(t, err) {
-		assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil")
-	}
-	assert.Nil(t, apima, "with closed repo, apima should be set back to nil")
-
+	basicTest(t, repo)
 }

node/repo/repo_test.go

@@ -0,0 +1,55 @@
+package repo
+
+import (
+	"testing"
+
+	"github.com/multiformats/go-multiaddr"
+	"github.com/stretchr/testify/assert"
+)
+
+func basicTest(t *testing.T, repo Repo) {
+	apima, err := repo.APIEndpoint()
+	if assert.Error(t, err) {
+		assert.Equal(t, ErrNoAPIEndpoint, err)
+	}
+	assert.Nil(t, apima, "with no api endpoint, return should be nil")
+
+	lrepo, err := repo.Lock()
+	assert.NoError(t, err, "should be able to lock once")
+	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
+
+	{
+		lrepo2, err := repo.Lock()
+		if assert.Error(t, err) {
+			assert.Equal(t, ErrRepoAlreadyLocked, err)
+		}
+		assert.Nil(t, lrepo2, "with locked repo errors, nil should be returned")
+	}
+
+	err = lrepo.Close()
+	assert.NoError(t, err, "should be able to unlock")
+
+	lrepo, err = repo.Lock()
+	assert.NoError(t, err, "should be able to relock")
+	assert.NotNil(t, lrepo, "locked repo shouldn't be nil")
+
+	ma, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/43244")
+	assert.NoError(t, err, "creating multiaddr shouldn't error")
+
+	err = lrepo.SetAPIEndpoint(ma)
+	assert.NoError(t, err, "setting multiaddr shouldn't error")
+
+	apima, err = repo.APIEndpoint()
+	assert.NoError(t, err, "setting multiaddr shouldn't error")
+	assert.Equal(t, ma, apima, "returned API multiaddr should be the same")
+
+	err = lrepo.Close()
+	assert.NoError(t, err, "should be able to close")
+
+	apima, err = repo.APIEndpoint()
+
+	if assert.Error(t, err) {
+		assert.Equal(t, ErrNoAPIEndpoint, err, "after closing repo, api should be nil")
+	}
+	assert.Nil(t, apima, "with closed repo, apima should be set back to nil")
+}