Merge pull request #2 from filecoin-project/feat/config

Add skeleton of a config
This commit is contained in:
Łukasz Magiera 2019-07-05 19:15:08 +02:00 committed by GitHub
commit 984f896473
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 427 additions and 124 deletions

View File

@ -17,7 +17,7 @@ workflows:
ci:
jobs:
- go/lint:
executor: go/circleci-golang
golangci-lint-version: 1.17.1
- go/test:
executor: go/circleci-golang
codecov-upload: true

View File

@ -2,7 +2,7 @@ linters:
disable-all: true
enable:
- vet
- gofmt
- goimports
- misspell
- goconst
- golint
@ -13,11 +13,13 @@ linters:
- varcheck
- structcheck
- deadcode
- scopelint
issues:
exclude:
- "func name will be used as test\\.Test.* by other packages, and that stutters; consider calling this"
- "Potential file inclusion via variable"
exclude-use-default: false
exclude-rules:

View File

@ -3,9 +3,10 @@ package daemon
import (
"context"
"gopkg.in/urfave/cli.v2"
"github.com/filecoin-project/go-lotus/node"
"github.com/filecoin-project/go-lotus/node/config"
"gopkg.in/urfave/cli.v2"
)
// Cmd is the `go-lotus daemon` command
@ -15,7 +16,12 @@ var Cmd = &cli.Command{
Action: func(cctx *cli.Context) error {
ctx := context.Background()
api, err := node.New(ctx)
cfg, err := config.FromFile("./config.toml")
if err != nil {
return err
}
api, err := node.New(ctx, node.Online(), node.Config(cfg))
if err != nil {
return err
}

5
go.mod
View File

@ -3,6 +3,9 @@ module github.com/filecoin-project/go-lotus
go 1.12
require (
github.com/BurntSushi/toml v0.3.1
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/ipfs/go-datastore v0.0.5
github.com/ipfs/go-ipfs-routing v0.1.0
github.com/ipfs/go-log v0.0.2-0.20190703113630-0c3cfb1eccc4
@ -25,9 +28,11 @@ require (
github.com/libp2p/go-maddr-filter v0.0.4
github.com/multiformats/go-multiaddr v0.0.4
github.com/multiformats/go-multihash v0.0.5
github.com/stretchr/testify v1.3.0
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
go.uber.org/dig v1.7.0 // indirect
go.uber.org/fx v1.9.0
go.uber.org/goleak v0.10.0 // indirect
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8
gopkg.in/yaml.v2 v2.2.2 // indirect
)

7
go.sum
View File

@ -1,5 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
@ -22,6 +23,8 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
@ -40,6 +43,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -364,4 +369,6 @@ gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0Y
gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -2,75 +2,189 @@ package node
import (
"context"
"errors"
"reflect"
"time"
"github.com/ipfs/go-datastore"
ci "github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/peerstore"
"github.com/libp2p/go-libp2p-core/routing"
"github.com/libp2p/go-libp2p-peerstore/pstoremem"
record "github.com/libp2p/go-libp2p-record"
"go.uber.org/fx"
"github.com/filecoin-project/go-lotus/api"
"github.com/filecoin-project/go-lotus/build"
"github.com/filecoin-project/go-lotus/node/config"
"github.com/filecoin-project/go-lotus/node/modules"
"github.com/filecoin-project/go-lotus/node/modules/helpers"
"github.com/filecoin-project/go-lotus/node/modules/lp2p"
)
var defaultListenAddrs = []string{ // TODO: better defaults?
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
// special is a type used to give keys to modules which
// can't really be identified by the returned type
type special struct{ id int }
//nolint:golint
var (
DefaultTransportsKey = special{0} // Libp2p option
PNetKey = special{1} // Option + multiret
DiscoveryHandlerKey = special{2} // Private type
AddrsFactoryKey = special{3} // Libp2p option
SmuxTransportKey = special{4} // Libp2p option
RelayKey = special{5} // Libp2p option
SecurityKey = special{6} // Libp2p option
BaseRoutingKey = special{7} // fx groups + multiret
NatPortMapKey = special{8} // Libp2p option
ConnectionManagerKey = special{9} // Libp2p option
)
type invoke int
const (
// PstoreAddSelfKeysKey is a key for Override for PstoreAddSelfKeys
PstoreAddSelfKeysKey = invoke(iota)
// StartListeningKey is a key for Override for StartListening
StartListeningKey
_nInvokes // keep this last
)
type settings struct {
// modules is a map of constructors for DI
//
// In most cases the index will be a reflect. Type of element returned by
// the constructor, but for some 'constructors' it's hard to specify what's
// the return type should be (or the constructor returns fx group)
modules map[interface{}]fx.Option
// invokes are separate from modules as they can't be referenced by return
// type, and must be applied in correct order
invokes []fx.Option
online bool // Online option applied
config bool // Config option applied
}
// Override option changes constructor for a given type
func Override(typ, constructor interface{}) Option {
return func(s *settings) error {
if i, ok := typ.(invoke); ok {
s.invokes[i] = fx.Invoke(constructor)
return nil
}
if c, ok := typ.(special); ok {
s.modules[c] = fx.Provide(constructor)
return nil
}
ctor := as(constructor, typ)
rt := reflect.TypeOf(typ).Elem()
s.modules[rt] = fx.Provide(ctor)
return nil
}
}
var defConf = config.Default()
var defaults = []Option{
Override(new(helpers.MetricsCtx), context.Background),
randomIdentity(),
Override(new(datastore.Batching), datastore.NewMapDatastore),
Override(new(record.Validator), modules.RecordValidator),
}
// Online sets up basic libp2p node
func Online() Option {
return Options(
// make sure that online is applied before Config.
// This is important because Config overrides some of Online units
func(s *settings) error { s.online = true; return nil },
applyIf(func(s *settings) bool { return s.config },
Error(errors.New("the Online option must be set before Config option")),
),
Override(new(peerstore.Peerstore), pstoremem.NewPeerstore),
Override(DefaultTransportsKey, lp2p.DefaultTransports),
Override(PNetKey, lp2p.PNet),
Override(new(lp2p.RawHost), lp2p.Host),
Override(new(host.Host), lp2p.RoutedHost),
Override(new(lp2p.BaseIpfsRouting), lp2p.DHTRouting(false)),
Override(DiscoveryHandlerKey, lp2p.DiscoveryHandler),
Override(AddrsFactoryKey, lp2p.AddrsFactory(nil, nil)),
Override(SmuxTransportKey, lp2p.SmuxTransport(true)),
Override(RelayKey, lp2p.Relay(true, false)),
Override(SecurityKey, lp2p.Security(true, false)),
Override(BaseRoutingKey, lp2p.BaseRouting),
Override(new(routing.Routing), lp2p.Routing),
Override(NatPortMapKey, lp2p.NatPortMap),
Override(ConnectionManagerKey, lp2p.ConnectionManager(50, 200, 20*time.Second)),
Override(PstoreAddSelfKeysKey, lp2p.PstoreAddSelfKeys),
Override(StartListeningKey, lp2p.StartListening(defConf.Libp2p.ListenAddresses)),
)
}
// Config sets up constructors based on the provided config
func Config(cfg *config.Root) Option {
return Options(
func(s *settings) error { s.config = true; return nil },
applyIf(func(s *settings) bool { return s.online },
Override(StartListeningKey, lp2p.StartListening(cfg.Libp2p.ListenAddresses)),
),
)
}
// New builds and starts new Filecoin node
func New(ctx context.Context) (api.API, error) {
func New(ctx context.Context, opts ...Option) (api.API, error) {
var resAPI api.Struct
settings := settings{
modules: map[interface{}]fx.Option{},
invokes: make([]fx.Option, _nInvokes),
}
online := true
// apply module options in the right order
if err := Options(Options(defaults...), Options(opts...))(&settings); err != nil {
return nil, err
}
// gather constructors for fx.Options
ctors := make([]fx.Option, 0, len(settings.modules))
for _, opt := range settings.modules {
ctors = append(ctors, opt)
}
// fill holes in invokes for use in fx.Options
for i, opt := range settings.invokes {
if opt == nil {
settings.invokes[i] = fx.Options()
}
}
app := fx.New(
fx.Provide(as(ctx, new(helpers.MetricsCtx))),
//fx.Provide(modules.RandomPeerID),
randomIdentity(),
memrepo(),
fx.Provide(modules.RecordValidator),
ifOpt(online,
fx.Provide(
pstoremem.NewPeerstore,
lp2p.DefaultTransports,
lp2p.PNet,
lp2p.Host,
lp2p.RoutedHost,
lp2p.DHTRouting(false),
lp2p.DiscoveryHandler,
lp2p.AddrsFactory(nil, nil),
lp2p.SmuxTransport(true),
lp2p.Relay(true, false),
lp2p.Security(true, false),
lp2p.BaseRouting,
lp2p.Routing,
lp2p.NatPortMap,
lp2p.ConnectionManager(50, 200, 20*time.Second),
),
fx.Invoke(
lp2p.PstoreAddSelfKeys,
lp2p.StartListening(defaultListenAddrs),
),
),
fx.Options(ctors...),
fx.Options(settings.invokes...),
fx.Invoke(versionAPI(&resAPI.Internal.Version)),
fx.Invoke(idAPI(&resAPI.Internal.ID)),
)
// TODO: we probably should have a 'firewall' for Closing signal
// on this context, and implement closing logic through lifecycles
// correctly
if err := app.Start(ctx); err != nil {
return nil, err
}
@ -80,36 +194,19 @@ func New(ctx context.Context) (api.API, error) {
// In-memory / testing
func memrepo() fx.Option {
return fx.Provide(
func() datastore.Batching {
return datastore.NewMapDatastore()
},
)
}
func randomIdentity() fx.Option {
func randomIdentity() Option {
sk, pk, err := ci.GenerateKeyPair(ci.RSA, 512)
if err != nil {
return fx.Error(err)
return Error(err)
}
return fx.Options(
fx.Provide(as(sk, new(ci.PrivKey))),
fx.Provide(as(pk, new(ci.PubKey))),
fx.Provide(peer.IDFromPublicKey),
return Options(
Override(new(ci.PrivKey), sk),
Override(new(ci.PubKey), pk),
Override(new(peer.ID), peer.IDFromPublicKey),
)
}
// UTILS
func ifOpt(cond bool, options ...fx.Option) fx.Option {
if cond {
return fx.Options(options...)
}
return fx.Options()
}
// API IMPL
// TODO: figure out a better way, this isn't usable in long term
@ -130,56 +227,3 @@ func versionAPI(set *func(context.Context) (api.Version, error)) func() {
}
}
}
// from go-ipfs
// as casts input constructor to a given interface (if a value is given, it
// wraps it into a constructor).
//
// Note: this method may look like a hack, and in fact it is one.
// This is here only because https://github.com/uber-go/fx/issues/673 wasn't
// released yet
//
// Note 2: when making changes here, make sure this method stays at
// 100% coverage. This makes it less likely it will be terribly broken
func as(in interface{}, as interface{}) interface{} {
outType := reflect.TypeOf(as)
if outType.Kind() != reflect.Ptr {
panic("outType is not a pointer")
}
if reflect.TypeOf(in).Kind() != reflect.Func {
ctype := reflect.FuncOf(nil, []reflect.Type{outType.Elem()}, false)
return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) {
out := reflect.New(outType.Elem())
out.Elem().Set(reflect.ValueOf(in))
return []reflect.Value{out.Elem()}
}).Interface()
}
inType := reflect.TypeOf(in)
ins := make([]reflect.Type, inType.NumIn())
outs := make([]reflect.Type, inType.NumOut())
for i := range ins {
ins[i] = inType.In(i)
}
outs[0] = outType.Elem()
for i := range outs[1:] {
outs[i+1] = inType.Out(i + 1)
}
ctype := reflect.FuncOf(ins, outs, false)
return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) {
outs := reflect.ValueOf(in).Call(args)
out := reflect.New(outType.Elem())
out.Elem().Set(outs[0])
outs[0] = out.Elem()
return outs
}).Interface()
}

50
node/config/def.go Normal file
View File

@ -0,0 +1,50 @@
package config
import "time"
// Root is starting point of the config
type Root struct {
API API
Libp2p Libp2p
}
// API contains configs for API endpoint
type API struct {
ListenAddress string
Timeout Duration
}
// Libp2p contains configs for libp2p
type Libp2p struct {
ListenAddresses []string
}
// Default returns the default config
func Default() *Root {
def := Root{
API: API{
ListenAddress: "/ip6/::1/tcp/1234/http",
Timeout: Duration(30 * time.Second),
},
Libp2p: Libp2p{
ListenAddresses: []string{
"/ip4/0.0.0.0/tcp/4001",
"/ip6/::/tcp/4001",
},
},
}
return &def
}
// Duration is a wrapper type for time.Duration for decoding it from TOML
type Duration time.Duration
// UnmarshalText implements interface for TOML decoding
func (dur *Duration) UnmarshalText(text []byte) error {
d, err := time.ParseDuration(string(text))
if err != nil {
return err
}
*dur = Duration(d)
return err
}

34
node/config/load.go Normal file
View File

@ -0,0 +1,34 @@
package config
import (
"io"
"os"
"github.com/BurntSushi/toml"
)
// FromFile loads config from a specified file overriding defaults specified in
// the source code. If file does not exist or is empty defaults are asummed.
func FromFile(path string) (*Root, error) {
file, err := os.Open(path)
switch {
case os.IsNotExist(err):
return Default(), nil
case err != nil:
return nil, err
}
defer file.Close() //nolint:errcheck // The file is RO
return FromReader(file)
}
// FromReader loads config from a reader instance.
func FromReader(reader io.Reader) (*Root, error) {
cfg := Default()
_, err := toml.DecodeReader(reader, cfg)
if err != nil {
return nil, err
}
return cfg, nil
}

63
node/config/load_test.go Normal file
View File

@ -0,0 +1,63 @@
package config
import (
"bytes"
"io/ioutil"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestDecodeNothing(t *testing.T) {
assert := assert.New(t)
{
cfg, err := FromFile(os.DevNull)
assert.Nil(err, "error should be nil")
assert.Equal(Default(), cfg,
"config from empty file should be the same as default")
}
{
cfg, err := FromFile("./does-not-exist.toml")
assert.Nil(err, "error should be nil")
assert.Equal(Default(), cfg,
"config from not exisiting file should be the same as default")
}
}
func TestParitalConfig(t *testing.T) {
assert := assert.New(t)
cfgString := `
[API]
Timeout = "10s"
`
expected := Default()
expected.API.Timeout = Duration(10 * time.Second)
{
cfg, err := FromReader(bytes.NewReader([]byte(cfgString)))
assert.NoError(err, "error should be nil")
assert.Equal(expected, cfg,
"config from reader should contain changes")
}
{
f, err := ioutil.TempFile("", "config-*.toml")
fname := f.Name()
assert.NoError(err, "tmp file shold not error")
_, err = f.WriteString(cfgString)
assert.NoError(err, "writing to tmp file should not error")
err = f.Close()
assert.NoError(err, "closing tmp file should not error")
defer os.Remove(fname) //nolint:errcheck
cfg, err := FromFile(fname)
assert.Nil(err, "error should be nil")
assert.Equal(expected, cfg,
"config from reader should contain changes")
}
}

View File

@ -4,7 +4,7 @@ import (
"fmt"
"github.com/libp2p/go-libp2p"
host "github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/host"
p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic"
mafilter "github.com/libp2p/go-maddr-filter"
ma "github.com/multiformats/go-multiaddr"

View File

@ -5,7 +5,7 @@ import (
logging "github.com/ipfs/go-log"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p-connmgr"
connmgr "github.com/libp2p/go-libp2p-connmgr"
"github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/peerstore"

92
node/options.go Normal file
View File

@ -0,0 +1,92 @@
package node
import (
"reflect"
)
// Option is a functional option which can be used with the New function to
// change how the node is constructed
//
// Options are applied in sequence
type Option func(*settings) error
// Options groups multiple options into one
func Options(opts ...Option) Option {
return func(s *settings) error {
for _, opt := range opts {
if err := opt(s); err != nil {
return err
}
}
return nil
}
}
// Error is a special option which returns an error when applied
func Error(err error) Option {
return func(_ *settings) error {
return err
}
}
func applyIf(check func(s *settings) bool, opts ...Option) Option {
return func(s *settings) error {
if check(s) {
return Options(opts...)(s)
}
return nil
}
}
// from go-ipfs
// as casts input constructor to a given interface (if a value is given, it
// wraps it into a constructor).
//
// Note: this method may look like a hack, and in fact it is one.
// This is here only because https://github.com/uber-go/fx/issues/673 wasn't
// released yet
//
// Note 2: when making changes here, make sure this method stays at
// 100% coverage. This makes it less likely it will be terribly broken
func as(in interface{}, as interface{}) interface{} {
outType := reflect.TypeOf(as)
if outType.Kind() != reflect.Ptr {
panic("outType is not a pointer")
}
if reflect.TypeOf(in).Kind() != reflect.Func {
ctype := reflect.FuncOf(nil, []reflect.Type{outType.Elem()}, false)
return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) {
out := reflect.New(outType.Elem())
out.Elem().Set(reflect.ValueOf(in))
return []reflect.Value{out.Elem()}
}).Interface()
}
inType := reflect.TypeOf(in)
ins := make([]reflect.Type, inType.NumIn())
outs := make([]reflect.Type, inType.NumOut())
for i := range ins {
ins[i] = inType.In(i)
}
outs[0] = outType.Elem()
for i := range outs[1:] {
outs[i+1] = inType.Out(i + 1)
}
ctype := reflect.FuncOf(ins, outs, false)
return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) {
outs := reflect.ValueOf(in).Call(args)
out := reflect.New(outType.Elem())
out.Elem().Set(outs[0])
outs[0] = out.Elem()
return outs
}).Interface()
}