roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

cmd/bootnode, p2p: support for alternate mapped ports (#26359)

Browse Source 
This changes the port mapping procedure such that, when the requested port is unavailable
an alternative port suggested by the router is used instead.

We now also repeatedly request the external IP from the router in order to catch any IP changes.

Co-authored-by: Felix Lange <fjl@twurst.com>
This commit is contained in:
Seungbae Yu 2023-07-14 21:58:27 +09:00 committed by GitHub
parent c40ab6af72
commit 60ecf48dd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 446 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
cmd/bootnode/main.go
View File

@ -23,6 +23,7 @@ import (
"fmt"
"net"
"os"
"time"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/crypto"
@ -108,20 +109,18 @@ func main() {
utils.Fatalf("-ListenUDP: %v", err)
}
realaddr := conn.LocalAddr().(*net.UDPAddr)
if natm != nil {
if !realaddr.IP.IsLoopback() {
go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
}
if ext, err := natm.ExternalIP(); err == nil {
realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
db, _ := enode.OpenDB("")
ln := enode.NewLocalNode(db, nodeKey)
listenerAddr := conn.LocalAddr().(*net.UDPAddr)
if natm != nil && !listenerAddr.IP.IsLoopback() {
natAddr := doPortMapping(natm, ln, listenerAddr)
if natAddr != nil {
listenerAddr = natAddr
}
}
printNotice(&nodeKey.PublicKey, *realaddr)
db, _ := enode.OpenDB("")
ln := enode.NewLocalNode(db, nodeKey)
printNotice(&nodeKey.PublicKey, *listenerAddr)
cfg := discover.Config{
PrivateKey: nodeKey,
NetRestrict: restrictList,
@ -148,3 +147,60 @@ func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) {
fmt.Println("Note: you're using cmd/bootnode, a developer tool.")
fmt.Println("We recommend using a regular node as bootstrap node for production deployments.")
}
func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) *net.UDPAddr {
const (
protocol = "udp"
name = "ethereum discovery"
)
newLogger := func(external int, internal int) log.Logger {
return log.New("proto", protocol, "extport", external, "intport", internal, "interface", natm)
}
var (
intport = addr.Port
extaddr = &net.UDPAddr{IP: addr.IP, Port: addr.Port}
mapTimeout = nat.DefaultMapTimeout
log = newLogger(addr.Port, intport)
)
addMapping := func() {
// Get the external address.
var err error
extaddr.IP, err = natm.ExternalIP()
if err != nil {
log.Debug("Couldn't get external IP", "err", err)
return
}
// Create the mapping.
p, err := natm.AddMapping(protocol, extaddr.Port, intport, name, mapTimeout)
if err != nil {
log.Debug("Couldn't add port mapping", "err", err)
return
}
if p != uint16(extaddr.Port) {
extaddr.Port = int(p)
log = newLogger(extaddr.Port, intport)
log.Info("NAT mapped alternative port")
} else {
log.Info("NAT mapped port")
}
// Update IP/port information of the local node.
ln.SetStaticIP(extaddr.IP)
ln.SetFallbackUDP(extaddr.Port)
}
// Perform mapping once, synchronously.
log.Info("Attempting port mapping")
addMapping()
// Refresh the mapping periodically.
go func() {
refresh := time.NewTimer(mapTimeout)
for range refresh.C {
addMapping()
refresh.Reset(mapTimeout)
}
}()
return extaddr
}

23
p2p/nat/nat.go
View File

@ -38,7 +38,7 @@ type Interface interface {
// protocol is "UDP" or "TCP". Some implementations allow setting
// a display name for the mapping. The mapping may be removed by
// the gateway when its lifetime ends.
AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error
AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error)
DeleteMapping(protocol string, extport, intport int) error
// ExternalIP should return the external (Internet-facing)
@ -91,20 +91,23 @@ func Parse(spec string) (Interface, error) {
}
const (
mapTimeout = 10 * time.Minute
DefaultMapTimeout = 10 * time.Minute
)
// Map adds a port mapping on m and keeps it alive until c is closed.
// This function is typically invoked in its own goroutine.
//
// Note that Map does not handle the situation where the NAT interface assigns a different
// external port than the requested one.
func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, name string) {
log := log.New("proto", protocol, "extport", extport, "intport", intport, "interface", m)
refresh := time.NewTimer(mapTimeout)
refresh := time.NewTimer(DefaultMapTimeout)
defer func() {
refresh.Stop()
log.Debug("Deleting port mapping")
m.DeleteMapping(protocol, extport, intport)
}()
if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil {
if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil {
log.Debug("Couldn't add port mapping", "err", err)
} else {
log.Info("Mapped network port")
@ -117,10 +120,10 @@ func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int,
}
case <-refresh.C:
log.Trace("Refreshing port mapping")
if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil {
if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil {
log.Debug("Couldn't add port mapping", "err", err)
}
refresh.Reset(mapTimeout)
refresh.Reset(DefaultMapTimeout)
}
}
}
@ -135,8 +138,8 @@ func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", ne
// These do nothing.
func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil }
func (ExtIP) DeleteMapping(string, int, int) error { return nil }
func (ExtIP) AddMapping(string, int, int, string, time.Duration) (uint16, error) { return 0, nil }
func (ExtIP) DeleteMapping(string, int, int) error { return nil }
// Any returns a port mapper that tries to discover any supported
// mechanism on the local network.
@ -193,9 +196,9 @@ func startautodisc(what string, doit func() Interface) Interface {
return &autodisc{what: what, doit: doit}
}
func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error {
func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) {
if err := n.wait(); err != nil {
return err
return 0, err
}
return n.found.AddMapping(protocol, extport, intport, name, lifetime)
}

21
p2p/nat/natpmp.go
View File

@ -44,28 +44,21 @@ func (n *pmp) ExternalIP() (net.IP, error) {
return response.ExternalIPAddress[:], nil
}
func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error {
func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) {
if lifetime <= 0 {
return fmt.Errorf("lifetime must not be <= 0")
return 0, fmt.Errorf("lifetime must not be <= 0")
}
// Note order of port arguments is switched between our
// AddMapping and the client's AddPortMapping.
res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second))
if err != nil {
return err
return 0, err
}
// NAT-PMP maps an alternative available port number if the requested
// port is already mapped to another address and returns success. In this
// case, we return an error because there is no way to return the new port
// to the caller.
if uint16(extport) != res.MappedExternalPort {
// Destroy the mapping in NAT device.
n.c.AddPortMapping(strings.ToLower(protocol), intport, 0, 0)
return fmt.Errorf("port %d already mapped to another address (%s)", extport, protocol)
}
return nil
// NAT-PMP maps an alternative available port number if the requested port
// is already mapped to another address and returns success. Handling of
// alternate port numbers is done by the caller.
return res.MappedExternalPort, nil
}
func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) {

41
p2p/nat/natupnp.go
View File

@ -19,6 +19,8 @@ package nat
import (
"errors"
"fmt"
"math"
"math/rand"
"net"
"strings"
"sync"
@ -40,6 +42,7 @@ type upnp struct {
client upnpClient
mu sync.Mutex
lastReqTime time.Time
rand *rand.Rand
}
type upnpClient interface {
@ -76,18 +79,50 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) {
return ip, nil
}
func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error {
func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) {
ip, err := n.internalAddress()
if err != nil {
return nil // TODO: Shouldn't we return the error?
return 0, nil // TODO: Shouldn't we return the error?
}
protocol = strings.ToUpper(protocol)
lifetimeS := uint32(lifetime / time.Second)
n.DeleteMapping(protocol, extport, intport)
return n.withRateLimit(func() error {
err = n.withRateLimit(func() error {
return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
})
if err == nil {
return uint16(extport), nil
}
return uint16(extport), n.withRateLimit(func() error {
p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS)
if err == nil {
extport = int(p)
}
return err
})
}
func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) {
if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok {
return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
}
// It will retry with a random port number if the client does
// not support AddAnyPortMapping.
extport = n.randomPort()
err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS)
if err != nil {
return 0, err
}
return uint16(extport), nil
}
func (n *upnp) randomPort() int {
if n.rand == nil {
n.rand = rand.New(rand.NewSource(time.Now().UnixNano()))
}
return n.rand.Intn(math.MaxUint16-10000) + 10000
}
func (n *upnp) internalAddress() (net.IP, error) {

57
p2p/server.go
View File

@ -195,6 +195,9 @@ type Server struct {
discmix *enode.FairMix
dialsched *dialScheduler
// This is read by the NAT port mapping loop.
portMappingRegister chan *portMapping
// Channels into the run loop.
quit chan struct{}
addtrusted chan *enode.Node
@ -483,6 +486,8 @@ func (srv *Server) Start() (err error) {
if err := srv.setupLocalNode(); err != nil {
return err
}
srv.setupPortMapping()
if srv.ListenAddr != "" {
if err := srv.setupListening(); err != nil {
return err
@ -521,24 +526,6 @@ func (srv *Server) setupLocalNode() error {
srv.localnode.Set(e)
}
}
switch srv.NAT.(type) {
case nil:
// No NAT interface, do nothing.
case nat.ExtIP:
// ExtIP doesn't block, set the IP right away.
ip, _ := srv.NAT.ExternalIP()
srv.localnode.SetStaticIP(ip)
default:
// Ask the router about the IP. This takes a while and blocks startup,
// do it in the background.
srv.loopWG.Add(1)
go func() {
defer srv.loopWG.Done()
if ip, err := srv.NAT.ExternalIP(); err == nil {
srv.localnode.SetStaticIP(ip)
}
}()
}
return nil
}
@ -656,14 +643,15 @@ func (srv *Server) setupListening() error {
srv.ListenAddr = listener.Addr().String()
// Update the local node record and map the TCP listening port if NAT is configured.
if tcp, ok := listener.Addr().(*net.TCPAddr); ok {
tcp, isTCP := listener.Addr().(*net.TCPAddr)
if isTCP {
srv.localnode.Set(enr.TCP(tcp.Port))
if !tcp.IP.IsLoopback() && srv.NAT != nil {
srv.loopWG.Add(1)
go func() {
nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p")
srv.loopWG.Done()
}()
if !tcp.IP.IsLoopback() && !tcp.IP.IsPrivate() {
srv.portMappingRegister <- &portMapping{
protocol: "TCP",
name: "ethereum p2p",
port: tcp.Port,
}
}
}
@ -688,18 +676,17 @@ func (srv *Server) setupUDPListening() (*net.UDPConn, error) {
if err != nil {
return nil, err
}
realaddr := conn.LocalAddr().(*net.UDPAddr)
srv.log.Debug("UDP listener up", "addr", realaddr)
if srv.NAT != nil {
if !realaddr.IP.IsLoopback() {
srv.loopWG.Add(1)
go func() {
nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
srv.loopWG.Done()
}()
laddr := conn.LocalAddr().(*net.UDPAddr)
srv.localnode.SetFallbackUDP(laddr.Port)
srv.log.Debug("UDP listener up", "addr", laddr)
if !laddr.IP.IsLoopback() && !laddr.IP.IsPrivate() {
srv.portMappingRegister <- &portMapping{
protocol: "UDP",
name: "ethereum peer discovery",
port: laddr.Port,
}
}
srv.localnode.SetFallbackUDP(realaddr.Port)
return conn, nil
}

187
p2p/server_nat.go Normal file
View File

@ -0,0 +1,187 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package p2p
import (
"net"
"time"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/enr"
"github.com/ethereum/go-ethereum/p2p/nat"
)
const (
portMapDuration = 10 * time.Minute
portMapRefreshInterval = 8 * time.Minute
portMapRetryInterval = 5 * time.Minute
extipRetryInterval = 2 * time.Minute
)
type portMapping struct {
protocol string
name string
port int
// for use by the portMappingLoop goroutine:
extPort int // the mapped port returned by the NAT interface
nextTime mclock.AbsTime
}
// setupPortMapping starts the port mapping loop if necessary.
// Note: this needs to be called after the LocalNode instance has been set on the server.
func (srv *Server) setupPortMapping() {
// portMappingRegister will receive up to two values: one for the TCP port if
// listening is enabled, and one more for enabling UDP port mapping if discovery is
// enabled. We make it buffered to avoid blocking setup while a mapping request is in
// progress.
srv.portMappingRegister = make(chan *portMapping, 2)
switch srv.NAT.(type) {
case nil:
// No NAT interface configured.
srv.loopWG.Add(1)
go srv.consumePortMappingRequests()
case nat.ExtIP:
// ExtIP doesn't block, set the IP right away.
ip, _ := srv.NAT.ExternalIP()
srv.localnode.SetStaticIP(ip)
srv.loopWG.Add(1)
go srv.consumePortMappingRequests()
default:
srv.loopWG.Add(1)
go srv.portMappingLoop()
}
}
func (srv *Server) consumePortMappingRequests() {
defer srv.loopWG.Done()
for {
select {
case <-srv.quit:
return
case <-srv.portMappingRegister:
}
}
}
// portMappingLoop manages port mappings for UDP and TCP.
func (srv *Server) portMappingLoop() {
defer srv.loopWG.Done()
newLogger := func(p string, e int, i int) log.Logger {
return log.New("proto", p, "extport", e, "intport", i, "interface", srv.NAT)
}
var (
mappings = make(map[string]*portMapping, 2)
refresh = mclock.NewAlarm(srv.clock)
extip = mclock.NewAlarm(srv.clock)
lastExtIP net.IP
)
extip.Schedule(srv.clock.Now())
defer func() {
refresh.Stop()
extip.Stop()
for _, m := range mappings {
if m.extPort != 0 {
log := newLogger(m.protocol, m.extPort, m.port)
log.Debug("Deleting port mapping")
srv.NAT.DeleteMapping(m.protocol, m.extPort, m.port)
}
}
}()
for {
// Schedule refresh of existing mappings.
for _, m := range mappings {
refresh.Schedule(m.nextTime)
}
select {
case <-srv.quit:
return
case <-extip.C():
extip.Schedule(srv.clock.Now().Add(extipRetryInterval))
ip, err := srv.NAT.ExternalIP()
if err != nil {
log.Debug("Couldn't get external IP", "err", err, "interface", srv.NAT)
} else if !ip.Equal(lastExtIP) {
log.Debug("External IP changed", "ip", extip, "interface", srv.NAT)
} else {
return
}
// Here, we either failed to get the external IP, or it has changed.
lastExtIP = ip
srv.localnode.SetStaticIP(ip)
// Ensure port mappings are refreshed in case we have moved to a new network.
for _, m := range mappings {
m.nextTime = srv.clock.Now()
}
case m := <-srv.portMappingRegister:
if m.protocol != "TCP" && m.protocol != "UDP" {
panic("unknown NAT protocol name: " + m.protocol)
}
mappings[m.protocol] = m
m.nextTime = srv.clock.Now()
case <-refresh.C():
for _, m := range mappings {
if srv.clock.Now() < m.nextTime {
continue
}
external := m.port
if m.extPort != 0 {
external = m.extPort
}
log := newLogger(m.protocol, external, m.port)
log.Trace("Attempting port mapping")
p, err := srv.NAT.AddMapping(m.protocol, external, m.port, m.name, portMapDuration)
if err != nil {
log.Debug("Couldn't add port mapping", "err", err)
m.extPort = 0
m.nextTime = srv.clock.Now().Add(portMapRetryInterval)
continue
}
// It was mapped!
m.extPort = int(p)
m.nextTime = srv.clock.Now().Add(portMapRefreshInterval)
if external != m.extPort {
log = newLogger(m.protocol, m.extPort, m.port)
log.Info("NAT mapped alternative port")
} else {
log.Info("NAT mapped port")
}
// Update port in local ENR.
switch m.protocol {
case "TCP":
srv.localnode.Set(enr.TCP(m.extPort))
case "UDP":
srv.localnode.SetFallbackUDP(m.extPort)
}
}
}
}
}

102
p2p/server_nat_test.go Normal file
View File

@ -0,0 +1,102 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package p2p
import (
"net"
"sync/atomic"
"testing"
"time"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/internal/testlog"
"github.com/ethereum/go-ethereum/log"
)
func TestServerPortMapping(t *testing.T) {
clock := new(mclock.Simulated)
mockNAT := &mockNAT{mappedPort: 30000}
srv := Server{
Config: Config{
PrivateKey: newkey(),
NoDial: true,
ListenAddr: ":0",
NAT: mockNAT,
Logger: testlog.Logger(t, log.LvlTrace),
clock: clock,
},
}
err := srv.Start()
if err != nil {
t.Fatal(err)
}
defer srv.Stop()
// Wait for the port mapping to be registered. Synchronization with the port mapping
// goroutine works like this: For each iteration, we allow other goroutines to run and
// also advance the virtual clock by 1 second. Waiting stops when the NAT interface
// has received some requests, or when the clock reaches a timeout.
deadline := clock.Now().Add(portMapRefreshInterval)
for clock.Now() < deadline && mockNAT.mapRequests.Load() < 2 {
time.Sleep(10 * time.Millisecond)
clock.Run(1 * time.Second)
}
if mockNAT.ipRequests.Load() == 0 {
t.Fatal("external IP was never requested")
}
reqCount := mockNAT.mapRequests.Load()
if reqCount != 2 {
t.Error("wrong request count:", reqCount)
}
enr := srv.LocalNode().Node()
if enr.IP().String() != "192.0.2.0" {
t.Error("wrong IP in ENR:", enr.IP())
}
if enr.TCP() != 30000 {
t.Error("wrong TCP port in ENR:", enr.TCP())
}
if enr.UDP() != 30000 {
t.Error("wrong UDP port in ENR:", enr.UDP())
}
}
type mockNAT struct {
mappedPort uint16
mapRequests atomic.Int32
unmapRequests atomic.Int32
ipRequests atomic.Int32
}
func (m *mockNAT) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) {
m.mapRequests.Add(1)
return m.mappedPort, nil
}
func (m *mockNAT) DeleteMapping(protocol string, extport, intport int) error {
m.unmapRequests.Add(1)
return nil
}
func (m *mockNAT) ExternalIP() (net.IP, error) {
m.ipRequests.Add(1)
return net.ParseIP("192.0.2.0"), nil
}
func (m *mockNAT) String() string {
return "mockNAT"
}

10
p2p/server_test.go
View File

@ -24,6 +24,8 @@ import (
"math/rand"
"net"
"reflect"
"strconv"
"strings"
"testing"
"time"
@ -224,6 +226,14 @@ func TestServerRemovePeerDisconnect(t *testing.T) {
srv2.Start()
defer srv2.Stop()
s := strings.Split(srv2.ListenAddr, ":")
if len(s) != 2 {
t.Fatal("invalid ListenAddr")
}
if port, err := strconv.Atoi(s[1]); err == nil {
srv2.localnode.Set(enr.TCP(uint16(port)))
}
if !syncAddPeer(srv1, srv2.Self()) {
t.Fatal("peer not connected")
}