diff --git a/README.md b/README.md index 3be57a2ec..dc1cd6ce9 100644 --- a/README.md +++ b/README.md @@ -36,10 +36,10 @@ Start full node daemon $ lotus daemon ``` -Connect to the network: +Check that you are connected to the network ```sh -$ lotus net connect /ip4/147.75.80.29/tcp/1347/p2p/12D3KooWGThG7Ct5aX4tTRkgvjr3pT2JyCyyvK77GhXVQ9Cfjzj2 -$ lotus net connect /ip4/147.75.80.17/tcp/1347/p2p/12D3KooWRNm4a6ESBr9bbTpSC2CfLfoWKRpABJi7FR3GhHw7usKW +$ lotus net peers | wc -l +2 # number of peers ``` [wait for the chain to finish syncing] diff --git a/build/bootstrap.go b/build/bootstrap.go new file mode 100644 index 000000000..94892d39b --- /dev/null +++ b/build/bootstrap.go @@ -0,0 +1,30 @@ +package build + +import ( + "context" + "github.com/filecoin-project/go-lotus/lib/addrutil" + "os" + "strings" + + rice "github.com/GeertJohan/go.rice" + "github.com/libp2p/go-libp2p-core/peer" +) + +func BuiltinBootstrap() ([]peer.AddrInfo, error) { + var out []peer.AddrInfo + + b := rice.MustFindBox("bootstrap") + err := b.Walk("", func(path string, info os.FileInfo, err error) error { + if !strings.HasSuffix(path, ".pi") { + return nil + } + spi := b.MustString(path) + if spi == "" { + return nil + } + pi, err := addrutil.ParseAddresses(context.TODO(), strings.Split(strings.TrimSpace(spi), "\n")) + out = append(out, pi...) + return err + }) + return out, err +} diff --git a/build/bootstrap/.gitkeep b/build/bootstrap/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/build/bootstrap/bootstrappers.pi b/build/bootstrap/bootstrappers.pi new file mode 100644 index 000000000..774d48bdb --- /dev/null +++ b/build/bootstrap/bootstrappers.pi @@ -0,0 +1 @@ +/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWK3mKvkdtNj7Bm9cJVufpiSnwQdkimuq5A69ZxFLxRvF1 diff --git a/build/bootstrap/root.pi b/build/bootstrap/root.pi new file mode 100644 index 000000000..ea7d8dc91 --- /dev/null +++ b/build/bootstrap/root.pi @@ -0,0 +1 @@ +/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWChZgMm5bmFF8ESr1QYonR8mPHaLkTzBafZ7iWnpWTtQX diff --git a/build/genesis/devnet.car b/build/genesis/devnet.car index 0484cf7d7..6add31af3 100644 Binary files a/build/genesis/devnet.car and b/build/genesis/devnet.car differ diff --git a/cli/net.go b/cli/net.go index b8cbad74e..b48b81413 100644 --- a/cli/net.go +++ b/cli/net.go @@ -1,15 +1,11 @@ package cli import ( - "context" "fmt" - "sync" - "time" - "github.com/libp2p/go-libp2p-core/peer" - ma "github.com/multiformats/go-multiaddr" - madns "github.com/multiformats/go-multiaddr-dns" "gopkg.in/urfave/cli.v2" + + "github.com/filecoin-project/go-lotus/lib/addrutil" ) var netCmd = &cli.Command{ @@ -80,7 +76,7 @@ var netConnect = &cli.Command{ defer closer() ctx := ReqContext(cctx) - pis, err := parseAddresses(ctx, cctx.Args().Slice()) + pis, err := addrutil.ParseAddresses(ctx, cctx.Args().Slice()) if err != nil { return err } @@ -120,80 +116,3 @@ var netId = &cli.Command{ return nil }, } - -// parseAddresses is a function that takes in a slice of string peer addresses -// (multiaddr + peerid) and returns a slice of properly constructed peers -func parseAddresses(ctx context.Context, addrs []string) ([]peer.AddrInfo, error) { - // resolve addresses - maddrs, err := resolveAddresses(ctx, addrs) - if err != nil { - return nil, err - } - - return peer.AddrInfosFromP2pAddrs(maddrs...) -} - -const ( - dnsResolveTimeout = 10 * time.Second -) - -// resolveAddresses resolves addresses parallelly -func resolveAddresses(ctx context.Context, addrs []string) ([]ma.Multiaddr, error) { - ctx, cancel := context.WithTimeout(ctx, dnsResolveTimeout) - defer cancel() - - var maddrs []ma.Multiaddr - var wg sync.WaitGroup - resolveErrC := make(chan error, len(addrs)) - - maddrC := make(chan ma.Multiaddr) - - for _, addr := range addrs { - maddr, err := ma.NewMultiaddr(addr) - if err != nil { - return nil, err - } - - // check whether address ends in `ipfs/Qm...` - if _, last := ma.SplitLast(maddr); last.Protocol().Code == ma.P_IPFS { - maddrs = append(maddrs, maddr) - continue - } - wg.Add(1) - go func(maddr ma.Multiaddr) { - defer wg.Done() - raddrs, err := madns.Resolve(ctx, maddr) - if err != nil { - resolveErrC <- err - return - } - // filter out addresses that still doesn't end in `ipfs/Qm...` - found := 0 - for _, raddr := range raddrs { - if _, last := ma.SplitLast(raddr); last != nil && last.Protocol().Code == ma.P_IPFS { - maddrC <- raddr - found++ - } - } - if found == 0 { - resolveErrC <- fmt.Errorf("found no ipfs peers at %s", maddr) - } - }(maddr) - } - go func() { - wg.Wait() - close(maddrC) - }() - - for maddr := range maddrC { - maddrs = append(maddrs, maddr) - } - - select { - case err := <-resolveErrC: - return nil, err - default: - } - - return maddrs, nil -} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 68bf41b38..a65c5ae39 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -40,6 +40,10 @@ var DaemonCmd = &cli.Command{ Name: "genesis", Usage: "genesis file to use for first node run", }, + &cli.BoolFlag{ + Name: "bootstrap", + Value: true, + }, }, Action: func(cctx *cli.Context) error { ctx := context.Background() @@ -95,7 +99,32 @@ var DaemonCmd = &cli.Command{ return err } + go func() { + if !cctx.Bool("bootstrap") { + return + } + err := bootstrap(ctx, api) + if err != nil { + log.Error("Bootstrap failed: ", err) + } + }() + // TODO: properly parse api endpoint (or make it a URL) return serveRPC(api, stop, "127.0.0.1:"+cctx.String("api")) }, } + +func bootstrap(ctx context.Context, api api.FullNode) error { + pis, err := build.BuiltinBootstrap() + if err != nil { + return err + } + + for _, pi := range pis { + if err := api.NetConnect(ctx, pi); err != nil { + return err + } + } + + return nil +} diff --git a/lib/addrutil/parse.go b/lib/addrutil/parse.go new file mode 100644 index 000000000..fad389e33 --- /dev/null +++ b/lib/addrutil/parse.go @@ -0,0 +1,89 @@ +package addrutil + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/libp2p/go-libp2p-core/peer" + ma "github.com/multiformats/go-multiaddr" + madns "github.com/multiformats/go-multiaddr-dns" +) + +// parseAddresses is a function that takes in a slice of string peer addresses +// (multiaddr + peerid) and returns a slice of properly constructed peers +func ParseAddresses(ctx context.Context, addrs []string) ([]peer.AddrInfo, error) { + // resolve addresses + maddrs, err := resolveAddresses(ctx, addrs) + if err != nil { + return nil, err + } + + return peer.AddrInfosFromP2pAddrs(maddrs...) +} + +const ( + dnsResolveTimeout = 10 * time.Second +) + +// resolveAddresses resolves addresses parallelly +func resolveAddresses(ctx context.Context, addrs []string) ([]ma.Multiaddr, error) { + ctx, cancel := context.WithTimeout(ctx, dnsResolveTimeout) + defer cancel() + + var maddrs []ma.Multiaddr + var wg sync.WaitGroup + resolveErrC := make(chan error, len(addrs)) + + maddrC := make(chan ma.Multiaddr) + + for _, addr := range addrs { + maddr, err := ma.NewMultiaddr(addr) + if err != nil { + return nil, err + } + + // check whether address ends in `ipfs/Qm...` + if _, last := ma.SplitLast(maddr); last.Protocol().Code == ma.P_IPFS { + maddrs = append(maddrs, maddr) + continue + } + wg.Add(1) + go func(maddr ma.Multiaddr) { + defer wg.Done() + raddrs, err := madns.Resolve(ctx, maddr) + if err != nil { + resolveErrC <- err + return + } + // filter out addresses that still doesn't end in `ipfs/Qm...` + found := 0 + for _, raddr := range raddrs { + if _, last := ma.SplitLast(raddr); last != nil && last.Protocol().Code == ma.P_IPFS { + maddrC <- raddr + found++ + } + } + if found == 0 { + resolveErrC <- fmt.Errorf("found no ipfs peers at %s", maddr) + } + }(maddr) + } + go func() { + wg.Wait() + close(maddrC) + }() + + for maddr := range maddrC { + maddrs = append(maddrs, maddr) + } + + select { + case err := <-resolveErrC: + return nil, err + default: + } + + return maddrs, nil +} diff --git a/node/builder.go b/node/builder.go index 39198a8ed..67c91345c 100644 --- a/node/builder.go +++ b/node/builder.go @@ -171,7 +171,7 @@ func libp2p() Option { Override(BaseRoutingKey, lp2p.BaseRouting), Override(new(routing.Routing), lp2p.Routing), - //Override(NatPortMapKey, lp2p.NatPortMap), //TODO: reenable when closing logic is actually there + Override(NatPortMapKey, lp2p.NatPortMap), Override(ConnectionManagerKey, lp2p.ConnectionManager(50, 200, 20*time.Second)), Override(new(*pubsub.PubSub), lp2p.GossipSub()), diff --git a/scripts/bootstrap.toml b/scripts/bootstrap.toml new file mode 100644 index 000000000..81cd60ccd --- /dev/null +++ b/scripts/bootstrap.toml @@ -0,0 +1,2 @@ +[Libp2p] +ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"] diff --git a/scripts/daemon.service b/scripts/daemon.service new file mode 100644 index 000000000..2055e4371 --- /dev/null +++ b/scripts/daemon.service @@ -0,0 +1,9 @@ +[Unit] +Description=Lotus Daemon +After=network.target + +[Service] +ExecStart=/usr/local/bin/lotus daemon + +[Install] +WantedBy=multiuser.target \ No newline at end of file diff --git a/scripts/deploy-devnet.sh b/scripts/deploy-devnet.sh new file mode 100755 index 000000000..f0c2f6809 --- /dev/null +++ b/scripts/deploy-devnet.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +############ +## Settings +GENESIS_HOST=root@147.75.80.29 +BOOTSTRAPPERS=( root@147.75.80.17 ) + +############ + +log() { + echo -e "\e[33m$1\e[39m" +} + +rm -f build/bootstrap/*.pi + +log '> Generating genesis' + +make + +GENPATH=$(mktemp -d) + +log 'staring temp daemon' + +./lotus --repo="${GENPATH}" daemon --lotus-make-random-genesis="${GENPATH}/devnet.car" & +GDPID=$! + +sleep 3 + +log 'Extracting genesis miner prvate key' + +ADDR=$(./lotus --repo="${GENPATH}" wallet list) +./lotus --repo="${GENPATH}" wallet export "$ADDR" > "${GENPATH}/wallet.key" + +kill "$GDPID" + +wait + +log '> Creating genesis binary' +cp "${GENPATH}/devnet.car" build/genesis/devnet.car +rm -f build/bootstrap/*.pi + +make + +log '> Deploying and starting genesis miner' + +ssh $GENESIS_HOST 'systemctl stop lotus-daemon' & +ssh $GENESIS_HOST 'systemctl stop lotus-storage-miner' & + +wait + +ssh $GENESIS_HOST 'rm -rf .lotus' & +ssh $GENESIS_HOST 'rm -rf .lotusstorage' & + +scp -C lotus "${GENESIS_HOST}":/usr/local/bin/lotus & +scp -C lotus-storage-miner "${GENESIS_HOST}":/usr/local/bin/lotus-storage-miner & + +wait + +log 'Initializing genesis miner repo' + +ssh $GENESIS_HOST 'systemctl start lotus-daemon' + +scp scripts/bootstrap.toml "${GENESIS_HOST}:.lotus/config.toml" & +ssh < "${GENPATH}/wallet.key" $GENESIS_HOST '/usr/local/bin/lotus wallet import' & +wait + +ssh $GENESIS_HOST 'systemctl restart lotus-daemon' + +log 'Starting genesis mining' + +ssh $GENESIS_HOST '/usr/local/bin/lotus-storage-miner init --genesis-miner --actor=t0101' +ssh $GENESIS_HOST 'systemctl start lotus-storage-miner' + + +log 'Getting genesis addr info' + +ssh $GENESIS_HOST './lotus net listen' | grep -v '/10' | grep -v '/127' > build/bootstrap/root.pi + +log '> Creating bootstrap binaries' +make + + +for host in "${BOOTSTRAPPERS[@]}" +do + log "> Deploying bootstrap node $host" + log "Stopping lotus daemon" + + ssh "$host" 'systemctl stop lotus-daemon' & + ssh "$host" 'systemctl stop lotus-storage-miner' & + + wait + + ssh "$host" 'rm -rf .lotus' & + ssh "$host" 'rm -rf .lotusstorage' & + + scp -C lotus "${host}":/usr/local/bin/lotus & + scp -C lotus-storage-miner "${host}":/usr/local/bin/lotus-storage-miner & + + wait + + log 'Initializing repo' + + ssh "$host" 'systemctl start lotus-daemon' + scp scripts/bootstrap.toml "${host}:.lotus/config.toml" + ssh "$host" 'systemctl restart lotus-daemon' + + log 'Extracting addr info' + + ssh "$host" './lotus net listen' | grep -v '/10' | grep -v '/127' >> build/bootstrap/bootstrappers.pi +done diff --git a/scripts/setup-host.sh b/scripts/setup-host.sh new file mode 100755 index 000000000..9372e1678 --- /dev/null +++ b/scripts/setup-host.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +HOST=$1 + +scp scripts/daemon.service "${HOST}:/etc/systemd/system/lotus-daemon.service" +scp scripts/sminer.service "${HOST}:/etc/systemd/system/lotus-storage-miner.service" diff --git a/scripts/sminer.service b/scripts/sminer.service new file mode 100644 index 000000000..c5453ef06 --- /dev/null +++ b/scripts/sminer.service @@ -0,0 +1,9 @@ +[Unit] +Description=Lotus Storage Miner +After=network.target + +[Service] +ExecStart=/usr/local/bin/lotus-storage-miner run + +[Install] +WantedBy=multiuser.target