From 60ecf48dd4496ad3576f25b616a091dee2ca4916 Mon Sep 17 00:00:00 2001 From: Seungbae Yu Date: Fri, 14 Jul 2023 21:58:27 +0900 Subject: [PATCH] cmd/bootnode, p2p: support for alternate mapped ports (#26359) This changes the port mapping procedure such that, when the requested port is unavailable an alternative port suggested by the router is used instead. We now also repeatedly request the external IP from the router in order to catch any IP changes. Co-authored-by: Felix Lange --- cmd/bootnode/main.go | 78 ++++++++++++++--- p2p/nat/nat.go | 23 ++--- p2p/nat/natpmp.go | 21 ++--- p2p/nat/natupnp.go | 41 ++++++++- p2p/server.go | 57 +++++-------- p2p/server_nat.go | 187 +++++++++++++++++++++++++++++++++++++++++ p2p/server_nat_test.go | 102 ++++++++++++++++++++++ p2p/server_test.go | 10 +++ 8 files changed, 446 insertions(+), 73 deletions(-) create mode 100644 p2p/server_nat.go create mode 100644 p2p/server_nat_test.go diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 748113aa4..bddfcbe2a 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "os" + "time" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" @@ -108,20 +109,18 @@ func main() { utils.Fatalf("-ListenUDP: %v", err) } - realaddr := conn.LocalAddr().(*net.UDPAddr) - if natm != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - if ext, err := natm.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) + + listenerAddr := conn.LocalAddr().(*net.UDPAddr) + if natm != nil && !listenerAddr.IP.IsLoopback() { + natAddr := doPortMapping(natm, ln, listenerAddr) + if natAddr != nil { + listenerAddr = natAddr } } - printNotice(&nodeKey.PublicKey, *realaddr) - - db, _ := enode.OpenDB("") - ln := enode.NewLocalNode(db, nodeKey) + printNotice(&nodeKey.PublicKey, *listenerAddr) cfg := discover.Config{ PrivateKey: nodeKey, NetRestrict: restrictList, @@ -148,3 +147,60 @@ func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) { fmt.Println("Note: you're using cmd/bootnode, a developer tool.") fmt.Println("We recommend using a regular node as bootstrap node for production deployments.") } + +func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) *net.UDPAddr { + const ( + protocol = "udp" + name = "ethereum discovery" + ) + newLogger := func(external int, internal int) log.Logger { + return log.New("proto", protocol, "extport", external, "intport", internal, "interface", natm) + } + + var ( + intport = addr.Port + extaddr = &net.UDPAddr{IP: addr.IP, Port: addr.Port} + mapTimeout = nat.DefaultMapTimeout + log = newLogger(addr.Port, intport) + ) + addMapping := func() { + // Get the external address. + var err error + extaddr.IP, err = natm.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err) + return + } + // Create the mapping. + p, err := natm.AddMapping(protocol, extaddr.Port, intport, name, mapTimeout) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + return + } + if p != uint16(extaddr.Port) { + extaddr.Port = int(p) + log = newLogger(extaddr.Port, intport) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + // Update IP/port information of the local node. + ln.SetStaticIP(extaddr.IP) + ln.SetFallbackUDP(extaddr.Port) + } + + // Perform mapping once, synchronously. + log.Info("Attempting port mapping") + addMapping() + + // Refresh the mapping periodically. + go func() { + refresh := time.NewTimer(mapTimeout) + for range refresh.C { + addMapping() + refresh.Reset(mapTimeout) + } + }() + + return extaddr +} diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index ad4c36582..61b692298 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -38,7 +38,7 @@ type Interface interface { // protocol is "UDP" or "TCP". Some implementations allow setting // a display name for the mapping. The mapping may be removed by // the gateway when its lifetime ends. - AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error + AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) DeleteMapping(protocol string, extport, intport int) error // ExternalIP should return the external (Internet-facing) @@ -91,20 +91,23 @@ func Parse(spec string) (Interface, error) { } const ( - mapTimeout = 10 * time.Minute + DefaultMapTimeout = 10 * time.Minute ) // Map adds a port mapping on m and keeps it alive until c is closed. // This function is typically invoked in its own goroutine. +// +// Note that Map does not handle the situation where the NAT interface assigns a different +// external port than the requested one. func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, name string) { log := log.New("proto", protocol, "extport", extport, "intport", intport, "interface", m) - refresh := time.NewTimer(mapTimeout) + refresh := time.NewTimer(DefaultMapTimeout) defer func() { refresh.Stop() log.Debug("Deleting port mapping") m.DeleteMapping(protocol, extport, intport) }() - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } else { log.Info("Mapped network port") @@ -117,10 +120,10 @@ func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, } case <-refresh.C: log.Trace("Refreshing port mapping") - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } - refresh.Reset(mapTimeout) + refresh.Reset(DefaultMapTimeout) } } } @@ -135,8 +138,8 @@ func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", ne // These do nothing. -func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil } -func (ExtIP) DeleteMapping(string, int, int) error { return nil } +func (ExtIP) AddMapping(string, int, int, string, time.Duration) (uint16, error) { return 0, nil } +func (ExtIP) DeleteMapping(string, int, int) error { return nil } // Any returns a port mapper that tries to discover any supported // mechanism on the local network. @@ -193,9 +196,9 @@ func startautodisc(what string, doit func() Interface) Interface { return &autodisc{what: what, doit: doit} } -func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if err := n.wait(); err != nil { - return err + return 0, err } return n.found.AddMapping(protocol, extport, intport, name, lifetime) } diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 40f2aff44..97601c99d 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -44,28 +44,21 @@ func (n *pmp) ExternalIP() (net.IP, error) { return response.ExternalIPAddress[:], nil } -func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return fmt.Errorf("lifetime must not be <= 0") + return 0, fmt.Errorf("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second)) if err != nil { - return err + return 0, err } - // NAT-PMP maps an alternative available port number if the requested - // port is already mapped to another address and returns success. In this - // case, we return an error because there is no way to return the new port - // to the caller. - if uint16(extport) != res.MappedExternalPort { - // Destroy the mapping in NAT device. - n.c.AddPortMapping(strings.ToLower(protocol), intport, 0, 0) - return fmt.Errorf("port %d already mapped to another address (%s)", extport, protocol) - } - - return nil + // NAT-PMP maps an alternative available port number if the requested port + // is already mapped to another address and returns success. Handling of + // alternate port numbers is done by the caller. + return res.MappedExternalPort, nil } func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) { diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index a8de00e97..c90c4f3de 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -19,6 +19,8 @@ package nat import ( "errors" "fmt" + "math" + "math/rand" "net" "strings" "sync" @@ -40,6 +42,7 @@ type upnp struct { client upnpClient mu sync.Mutex lastReqTime time.Time + rand *rand.Rand } type upnpClient interface { @@ -76,18 +79,50 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { return ip, nil } -func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { +func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) { ip, err := n.internalAddress() if err != nil { - return nil // TODO: Shouldn't we return the error? + return 0, nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) n.DeleteMapping(protocol, extport, intport) - return n.withRateLimit(func() error { + err = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) + if err == nil { + return uint16(extport), nil + } + + return uint16(extport), n.withRateLimit(func() error { + p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) + if err == nil { + extport = int(p) + } + return err + }) +} + +func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { + if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok { + return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + } + // It will retry with a random port number if the client does + // not support AddAnyPortMapping. + extport = n.randomPort() + err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + if err != nil { + return 0, err + } + return uint16(extport), nil +} + +func (n *upnp) randomPort() int { + if n.rand == nil { + n.rand = rand.New(rand.NewSource(time.Now().UnixNano())) + } + return n.rand.Intn(math.MaxUint16-10000) + 10000 } func (n *upnp) internalAddress() (net.IP, error) { diff --git a/p2p/server.go b/p2p/server.go index bd0fd7095..d4e2be678 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -195,6 +195,9 @@ type Server struct { discmix *enode.FairMix dialsched *dialScheduler + // This is read by the NAT port mapping loop. + portMappingRegister chan *portMapping + // Channels into the run loop. quit chan struct{} addtrusted chan *enode.Node @@ -483,6 +486,8 @@ func (srv *Server) Start() (err error) { if err := srv.setupLocalNode(); err != nil { return err } + srv.setupPortMapping() + if srv.ListenAddr != "" { if err := srv.setupListening(); err != nil { return err @@ -521,24 +526,6 @@ func (srv *Server) setupLocalNode() error { srv.localnode.Set(e) } } - switch srv.NAT.(type) { - case nil: - // No NAT interface, do nothing. - case nat.ExtIP: - // ExtIP doesn't block, set the IP right away. - ip, _ := srv.NAT.ExternalIP() - srv.localnode.SetStaticIP(ip) - default: - // Ask the router about the IP. This takes a while and blocks startup, - // do it in the background. - srv.loopWG.Add(1) - go func() { - defer srv.loopWG.Done() - if ip, err := srv.NAT.ExternalIP(); err == nil { - srv.localnode.SetStaticIP(ip) - } - }() - } return nil } @@ -656,14 +643,15 @@ func (srv *Server) setupListening() error { srv.ListenAddr = listener.Addr().String() // Update the local node record and map the TCP listening port if NAT is configured. - if tcp, ok := listener.Addr().(*net.TCPAddr); ok { + tcp, isTCP := listener.Addr().(*net.TCPAddr) + if isTCP { srv.localnode.Set(enr.TCP(tcp.Port)) - if !tcp.IP.IsLoopback() && srv.NAT != nil { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p") - srv.loopWG.Done() - }() + if !tcp.IP.IsLoopback() && !tcp.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "TCP", + name: "ethereum p2p", + port: tcp.Port, + } } } @@ -688,18 +676,17 @@ func (srv *Server) setupUDPListening() (*net.UDPConn, error) { if err != nil { return nil, err } - realaddr := conn.LocalAddr().(*net.UDPAddr) - srv.log.Debug("UDP listener up", "addr", realaddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - srv.loopWG.Done() - }() + laddr := conn.LocalAddr().(*net.UDPAddr) + srv.localnode.SetFallbackUDP(laddr.Port) + srv.log.Debug("UDP listener up", "addr", laddr) + if !laddr.IP.IsLoopback() && !laddr.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "UDP", + name: "ethereum peer discovery", + port: laddr.Port, } } - srv.localnode.SetFallbackUDP(realaddr.Port) + return conn, nil } diff --git a/p2p/server_nat.go b/p2p/server_nat.go new file mode 100644 index 000000000..354597cc7 --- /dev/null +++ b/p2p/server_nat.go @@ -0,0 +1,187 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/nat" +) + +const ( + portMapDuration = 10 * time.Minute + portMapRefreshInterval = 8 * time.Minute + portMapRetryInterval = 5 * time.Minute + extipRetryInterval = 2 * time.Minute +) + +type portMapping struct { + protocol string + name string + port int + + // for use by the portMappingLoop goroutine: + extPort int // the mapped port returned by the NAT interface + nextTime mclock.AbsTime +} + +// setupPortMapping starts the port mapping loop if necessary. +// Note: this needs to be called after the LocalNode instance has been set on the server. +func (srv *Server) setupPortMapping() { + // portMappingRegister will receive up to two values: one for the TCP port if + // listening is enabled, and one more for enabling UDP port mapping if discovery is + // enabled. We make it buffered to avoid blocking setup while a mapping request is in + // progress. + srv.portMappingRegister = make(chan *portMapping, 2) + + switch srv.NAT.(type) { + case nil: + // No NAT interface configured. + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + case nat.ExtIP: + // ExtIP doesn't block, set the IP right away. + ip, _ := srv.NAT.ExternalIP() + srv.localnode.SetStaticIP(ip) + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + default: + srv.loopWG.Add(1) + go srv.portMappingLoop() + } +} + +func (srv *Server) consumePortMappingRequests() { + defer srv.loopWG.Done() + for { + select { + case <-srv.quit: + return + case <-srv.portMappingRegister: + } + } +} + +// portMappingLoop manages port mappings for UDP and TCP. +func (srv *Server) portMappingLoop() { + defer srv.loopWG.Done() + + newLogger := func(p string, e int, i int) log.Logger { + return log.New("proto", p, "extport", e, "intport", i, "interface", srv.NAT) + } + + var ( + mappings = make(map[string]*portMapping, 2) + refresh = mclock.NewAlarm(srv.clock) + extip = mclock.NewAlarm(srv.clock) + lastExtIP net.IP + ) + extip.Schedule(srv.clock.Now()) + defer func() { + refresh.Stop() + extip.Stop() + for _, m := range mappings { + if m.extPort != 0 { + log := newLogger(m.protocol, m.extPort, m.port) + log.Debug("Deleting port mapping") + srv.NAT.DeleteMapping(m.protocol, m.extPort, m.port) + } + } + }() + + for { + // Schedule refresh of existing mappings. + for _, m := range mappings { + refresh.Schedule(m.nextTime) + } + + select { + case <-srv.quit: + return + + case <-extip.C(): + extip.Schedule(srv.clock.Now().Add(extipRetryInterval)) + ip, err := srv.NAT.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err, "interface", srv.NAT) + } else if !ip.Equal(lastExtIP) { + log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) + } else { + return + } + // Here, we either failed to get the external IP, or it has changed. + lastExtIP = ip + srv.localnode.SetStaticIP(ip) + // Ensure port mappings are refreshed in case we have moved to a new network. + for _, m := range mappings { + m.nextTime = srv.clock.Now() + } + + case m := <-srv.portMappingRegister: + if m.protocol != "TCP" && m.protocol != "UDP" { + panic("unknown NAT protocol name: " + m.protocol) + } + mappings[m.protocol] = m + m.nextTime = srv.clock.Now() + + case <-refresh.C(): + for _, m := range mappings { + if srv.clock.Now() < m.nextTime { + continue + } + + external := m.port + if m.extPort != 0 { + external = m.extPort + } + log := newLogger(m.protocol, external, m.port) + + log.Trace("Attempting port mapping") + p, err := srv.NAT.AddMapping(m.protocol, external, m.port, m.name, portMapDuration) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + m.extPort = 0 + m.nextTime = srv.clock.Now().Add(portMapRetryInterval) + continue + } + // It was mapped! + m.extPort = int(p) + m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) + if external != m.extPort { + log = newLogger(m.protocol, m.extPort, m.port) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + + // Update port in local ENR. + switch m.protocol { + case "TCP": + srv.localnode.Set(enr.TCP(m.extPort)) + case "UDP": + srv.localnode.SetFallbackUDP(m.extPort) + } + } + } + } +} diff --git a/p2p/server_nat_test.go b/p2p/server_nat_test.go new file mode 100644 index 000000000..de935fcfc --- /dev/null +++ b/p2p/server_nat_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/internal/testlog" + "github.com/ethereum/go-ethereum/log" +) + +func TestServerPortMapping(t *testing.T) { + clock := new(mclock.Simulated) + mockNAT := &mockNAT{mappedPort: 30000} + srv := Server{ + Config: Config{ + PrivateKey: newkey(), + NoDial: true, + ListenAddr: ":0", + NAT: mockNAT, + Logger: testlog.Logger(t, log.LvlTrace), + clock: clock, + }, + } + err := srv.Start() + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + + // Wait for the port mapping to be registered. Synchronization with the port mapping + // goroutine works like this: For each iteration, we allow other goroutines to run and + // also advance the virtual clock by 1 second. Waiting stops when the NAT interface + // has received some requests, or when the clock reaches a timeout. + deadline := clock.Now().Add(portMapRefreshInterval) + for clock.Now() < deadline && mockNAT.mapRequests.Load() < 2 { + time.Sleep(10 * time.Millisecond) + clock.Run(1 * time.Second) + } + + if mockNAT.ipRequests.Load() == 0 { + t.Fatal("external IP was never requested") + } + reqCount := mockNAT.mapRequests.Load() + if reqCount != 2 { + t.Error("wrong request count:", reqCount) + } + enr := srv.LocalNode().Node() + if enr.IP().String() != "192.0.2.0" { + t.Error("wrong IP in ENR:", enr.IP()) + } + if enr.TCP() != 30000 { + t.Error("wrong TCP port in ENR:", enr.TCP()) + } + if enr.UDP() != 30000 { + t.Error("wrong UDP port in ENR:", enr.UDP()) + } +} + +type mockNAT struct { + mappedPort uint16 + mapRequests atomic.Int32 + unmapRequests atomic.Int32 + ipRequests atomic.Int32 +} + +func (m *mockNAT) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { + m.mapRequests.Add(1) + return m.mappedPort, nil +} + +func (m *mockNAT) DeleteMapping(protocol string, extport, intport int) error { + m.unmapRequests.Add(1) + return nil +} + +func (m *mockNAT) ExternalIP() (net.IP, error) { + m.ipRequests.Add(1) + return net.ParseIP("192.0.2.0"), nil +} + +func (m *mockNAT) String() string { + return "mockNAT" +} diff --git a/p2p/server_test.go b/p2p/server_test.go index c8bf4c941..a0491e984 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -24,6 +24,8 @@ import ( "math/rand" "net" "reflect" + "strconv" + "strings" "testing" "time" @@ -224,6 +226,14 @@ func TestServerRemovePeerDisconnect(t *testing.T) { srv2.Start() defer srv2.Stop() + s := strings.Split(srv2.ListenAddr, ":") + if len(s) != 2 { + t.Fatal("invalid ListenAddr") + } + if port, err := strconv.Atoi(s[1]); err == nil { + srv2.localnode.Set(enr.TCP(uint16(port))) + } + if !syncAddPeer(srv1, srv2.Self()) { t.Fatal("peer not connected") }