diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 748113aa48..bddfcbe2a5 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "os" + "time" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" @@ -108,20 +109,18 @@ func main() { utils.Fatalf("-ListenUDP: %v", err) } - realaddr := conn.LocalAddr().(*net.UDPAddr) - if natm != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - if ext, err := natm.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) + + listenerAddr := conn.LocalAddr().(*net.UDPAddr) + if natm != nil && !listenerAddr.IP.IsLoopback() { + natAddr := doPortMapping(natm, ln, listenerAddr) + if natAddr != nil { + listenerAddr = natAddr } } - printNotice(&nodeKey.PublicKey, *realaddr) - - db, _ := enode.OpenDB("") - ln := enode.NewLocalNode(db, nodeKey) + printNotice(&nodeKey.PublicKey, *listenerAddr) cfg := discover.Config{ PrivateKey: nodeKey, NetRestrict: restrictList, @@ -148,3 +147,60 @@ func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) { fmt.Println("Note: you're using cmd/bootnode, a developer tool.") fmt.Println("We recommend using a regular node as bootstrap node for production deployments.") } + +func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) *net.UDPAddr { + const ( + protocol = "udp" + name = "ethereum discovery" + ) + newLogger := func(external int, internal int) log.Logger { + return log.New("proto", protocol, "extport", external, "intport", internal, "interface", natm) + } + + var ( + intport = addr.Port + extaddr = &net.UDPAddr{IP: addr.IP, Port: addr.Port} + mapTimeout = nat.DefaultMapTimeout + log = newLogger(addr.Port, intport) + ) + addMapping := func() { + // Get the external address. + var err error + extaddr.IP, err = natm.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err) + return + } + // Create the mapping. + p, err := natm.AddMapping(protocol, extaddr.Port, intport, name, mapTimeout) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + return + } + if p != uint16(extaddr.Port) { + extaddr.Port = int(p) + log = newLogger(extaddr.Port, intport) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + // Update IP/port information of the local node. + ln.SetStaticIP(extaddr.IP) + ln.SetFallbackUDP(extaddr.Port) + } + + // Perform mapping once, synchronously. + log.Info("Attempting port mapping") + addMapping() + + // Refresh the mapping periodically. + go func() { + refresh := time.NewTimer(mapTimeout) + for range refresh.C { + addMapping() + refresh.Reset(mapTimeout) + } + }() + + return extaddr +} diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index ad4c36582a..61b6922988 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -38,7 +38,7 @@ type Interface interface { // protocol is "UDP" or "TCP". Some implementations allow setting // a display name for the mapping. The mapping may be removed by // the gateway when its lifetime ends. - AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error + AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) DeleteMapping(protocol string, extport, intport int) error // ExternalIP should return the external (Internet-facing) @@ -91,20 +91,23 @@ func Parse(spec string) (Interface, error) { } const ( - mapTimeout = 10 * time.Minute + DefaultMapTimeout = 10 * time.Minute ) // Map adds a port mapping on m and keeps it alive until c is closed. // This function is typically invoked in its own goroutine. +// +// Note that Map does not handle the situation where the NAT interface assigns a different +// external port than the requested one. func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, name string) { log := log.New("proto", protocol, "extport", extport, "intport", intport, "interface", m) - refresh := time.NewTimer(mapTimeout) + refresh := time.NewTimer(DefaultMapTimeout) defer func() { refresh.Stop() log.Debug("Deleting port mapping") m.DeleteMapping(protocol, extport, intport) }() - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } else { log.Info("Mapped network port") @@ -117,10 +120,10 @@ func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, } case <-refresh.C: log.Trace("Refreshing port mapping") - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } - refresh.Reset(mapTimeout) + refresh.Reset(DefaultMapTimeout) } } } @@ -135,8 +138,8 @@ func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", ne // These do nothing. -func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil } -func (ExtIP) DeleteMapping(string, int, int) error { return nil } +func (ExtIP) AddMapping(string, int, int, string, time.Duration) (uint16, error) { return 0, nil } +func (ExtIP) DeleteMapping(string, int, int) error { return nil } // Any returns a port mapper that tries to discover any supported // mechanism on the local network. @@ -193,9 +196,9 @@ func startautodisc(what string, doit func() Interface) Interface { return &autodisc{what: what, doit: doit} } -func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if err := n.wait(); err != nil { - return err + return 0, err } return n.found.AddMapping(protocol, extport, intport, name, lifetime) } diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 40f2aff44e..97601c99dc 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -44,28 +44,21 @@ func (n *pmp) ExternalIP() (net.IP, error) { return response.ExternalIPAddress[:], nil } -func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return fmt.Errorf("lifetime must not be <= 0") + return 0, fmt.Errorf("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second)) if err != nil { - return err + return 0, err } - // NAT-PMP maps an alternative available port number if the requested - // port is already mapped to another address and returns success. In this - // case, we return an error because there is no way to return the new port - // to the caller. - if uint16(extport) != res.MappedExternalPort { - // Destroy the mapping in NAT device. - n.c.AddPortMapping(strings.ToLower(protocol), intport, 0, 0) - return fmt.Errorf("port %d already mapped to another address (%s)", extport, protocol) - } - - return nil + // NAT-PMP maps an alternative available port number if the requested port + // is already mapped to another address and returns success. Handling of + // alternate port numbers is done by the caller. + return res.MappedExternalPort, nil } func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) { diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index a8de00e978..c90c4f3de8 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -19,6 +19,8 @@ package nat import ( "errors" "fmt" + "math" + "math/rand" "net" "strings" "sync" @@ -40,6 +42,7 @@ type upnp struct { client upnpClient mu sync.Mutex lastReqTime time.Time + rand *rand.Rand } type upnpClient interface { @@ -76,18 +79,50 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { return ip, nil } -func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { +func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) { ip, err := n.internalAddress() if err != nil { - return nil // TODO: Shouldn't we return the error? + return 0, nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) n.DeleteMapping(protocol, extport, intport) - return n.withRateLimit(func() error { + err = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) + if err == nil { + return uint16(extport), nil + } + + return uint16(extport), n.withRateLimit(func() error { + p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) + if err == nil { + extport = int(p) + } + return err + }) +} + +func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { + if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok { + return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + } + // It will retry with a random port number if the client does + // not support AddAnyPortMapping. + extport = n.randomPort() + err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + if err != nil { + return 0, err + } + return uint16(extport), nil +} + +func (n *upnp) randomPort() int { + if n.rand == nil { + n.rand = rand.New(rand.NewSource(time.Now().UnixNano())) + } + return n.rand.Intn(math.MaxUint16-10000) + 10000 } func (n *upnp) internalAddress() (net.IP, error) { diff --git a/p2p/server.go b/p2p/server.go index bd0fd70950..d4e2be6783 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -195,6 +195,9 @@ type Server struct { discmix *enode.FairMix dialsched *dialScheduler + // This is read by the NAT port mapping loop. + portMappingRegister chan *portMapping + // Channels into the run loop. quit chan struct{} addtrusted chan *enode.Node @@ -483,6 +486,8 @@ func (srv *Server) Start() (err error) { if err := srv.setupLocalNode(); err != nil { return err } + srv.setupPortMapping() + if srv.ListenAddr != "" { if err := srv.setupListening(); err != nil { return err @@ -521,24 +526,6 @@ func (srv *Server) setupLocalNode() error { srv.localnode.Set(e) } } - switch srv.NAT.(type) { - case nil: - // No NAT interface, do nothing. - case nat.ExtIP: - // ExtIP doesn't block, set the IP right away. - ip, _ := srv.NAT.ExternalIP() - srv.localnode.SetStaticIP(ip) - default: - // Ask the router about the IP. This takes a while and blocks startup, - // do it in the background. - srv.loopWG.Add(1) - go func() { - defer srv.loopWG.Done() - if ip, err := srv.NAT.ExternalIP(); err == nil { - srv.localnode.SetStaticIP(ip) - } - }() - } return nil } @@ -656,14 +643,15 @@ func (srv *Server) setupListening() error { srv.ListenAddr = listener.Addr().String() // Update the local node record and map the TCP listening port if NAT is configured. - if tcp, ok := listener.Addr().(*net.TCPAddr); ok { + tcp, isTCP := listener.Addr().(*net.TCPAddr) + if isTCP { srv.localnode.Set(enr.TCP(tcp.Port)) - if !tcp.IP.IsLoopback() && srv.NAT != nil { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p") - srv.loopWG.Done() - }() + if !tcp.IP.IsLoopback() && !tcp.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "TCP", + name: "ethereum p2p", + port: tcp.Port, + } } } @@ -688,18 +676,17 @@ func (srv *Server) setupUDPListening() (*net.UDPConn, error) { if err != nil { return nil, err } - realaddr := conn.LocalAddr().(*net.UDPAddr) - srv.log.Debug("UDP listener up", "addr", realaddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - srv.loopWG.Done() - }() + laddr := conn.LocalAddr().(*net.UDPAddr) + srv.localnode.SetFallbackUDP(laddr.Port) + srv.log.Debug("UDP listener up", "addr", laddr) + if !laddr.IP.IsLoopback() && !laddr.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "UDP", + name: "ethereum peer discovery", + port: laddr.Port, } } - srv.localnode.SetFallbackUDP(realaddr.Port) + return conn, nil } diff --git a/p2p/server_nat.go b/p2p/server_nat.go new file mode 100644 index 0000000000..354597cc7a --- /dev/null +++ b/p2p/server_nat.go @@ -0,0 +1,187 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/nat" +) + +const ( + portMapDuration = 10 * time.Minute + portMapRefreshInterval = 8 * time.Minute + portMapRetryInterval = 5 * time.Minute + extipRetryInterval = 2 * time.Minute +) + +type portMapping struct { + protocol string + name string + port int + + // for use by the portMappingLoop goroutine: + extPort int // the mapped port returned by the NAT interface + nextTime mclock.AbsTime +} + +// setupPortMapping starts the port mapping loop if necessary. +// Note: this needs to be called after the LocalNode instance has been set on the server. +func (srv *Server) setupPortMapping() { + // portMappingRegister will receive up to two values: one for the TCP port if + // listening is enabled, and one more for enabling UDP port mapping if discovery is + // enabled. We make it buffered to avoid blocking setup while a mapping request is in + // progress. + srv.portMappingRegister = make(chan *portMapping, 2) + + switch srv.NAT.(type) { + case nil: + // No NAT interface configured. + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + case nat.ExtIP: + // ExtIP doesn't block, set the IP right away. + ip, _ := srv.NAT.ExternalIP() + srv.localnode.SetStaticIP(ip) + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + default: + srv.loopWG.Add(1) + go srv.portMappingLoop() + } +} + +func (srv *Server) consumePortMappingRequests() { + defer srv.loopWG.Done() + for { + select { + case <-srv.quit: + return + case <-srv.portMappingRegister: + } + } +} + +// portMappingLoop manages port mappings for UDP and TCP. +func (srv *Server) portMappingLoop() { + defer srv.loopWG.Done() + + newLogger := func(p string, e int, i int) log.Logger { + return log.New("proto", p, "extport", e, "intport", i, "interface", srv.NAT) + } + + var ( + mappings = make(map[string]*portMapping, 2) + refresh = mclock.NewAlarm(srv.clock) + extip = mclock.NewAlarm(srv.clock) + lastExtIP net.IP + ) + extip.Schedule(srv.clock.Now()) + defer func() { + refresh.Stop() + extip.Stop() + for _, m := range mappings { + if m.extPort != 0 { + log := newLogger(m.protocol, m.extPort, m.port) + log.Debug("Deleting port mapping") + srv.NAT.DeleteMapping(m.protocol, m.extPort, m.port) + } + } + }() + + for { + // Schedule refresh of existing mappings. + for _, m := range mappings { + refresh.Schedule(m.nextTime) + } + + select { + case <-srv.quit: + return + + case <-extip.C(): + extip.Schedule(srv.clock.Now().Add(extipRetryInterval)) + ip, err := srv.NAT.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err, "interface", srv.NAT) + } else if !ip.Equal(lastExtIP) { + log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) + } else { + return + } + // Here, we either failed to get the external IP, or it has changed. + lastExtIP = ip + srv.localnode.SetStaticIP(ip) + // Ensure port mappings are refreshed in case we have moved to a new network. + for _, m := range mappings { + m.nextTime = srv.clock.Now() + } + + case m := <-srv.portMappingRegister: + if m.protocol != "TCP" && m.protocol != "UDP" { + panic("unknown NAT protocol name: " + m.protocol) + } + mappings[m.protocol] = m + m.nextTime = srv.clock.Now() + + case <-refresh.C(): + for _, m := range mappings { + if srv.clock.Now() < m.nextTime { + continue + } + + external := m.port + if m.extPort != 0 { + external = m.extPort + } + log := newLogger(m.protocol, external, m.port) + + log.Trace("Attempting port mapping") + p, err := srv.NAT.AddMapping(m.protocol, external, m.port, m.name, portMapDuration) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + m.extPort = 0 + m.nextTime = srv.clock.Now().Add(portMapRetryInterval) + continue + } + // It was mapped! + m.extPort = int(p) + m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) + if external != m.extPort { + log = newLogger(m.protocol, m.extPort, m.port) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + + // Update port in local ENR. + switch m.protocol { + case "TCP": + srv.localnode.Set(enr.TCP(m.extPort)) + case "UDP": + srv.localnode.SetFallbackUDP(m.extPort) + } + } + } + } +} diff --git a/p2p/server_nat_test.go b/p2p/server_nat_test.go new file mode 100644 index 0000000000..de935fcfc5 --- /dev/null +++ b/p2p/server_nat_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/internal/testlog" + "github.com/ethereum/go-ethereum/log" +) + +func TestServerPortMapping(t *testing.T) { + clock := new(mclock.Simulated) + mockNAT := &mockNAT{mappedPort: 30000} + srv := Server{ + Config: Config{ + PrivateKey: newkey(), + NoDial: true, + ListenAddr: ":0", + NAT: mockNAT, + Logger: testlog.Logger(t, log.LvlTrace), + clock: clock, + }, + } + err := srv.Start() + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + + // Wait for the port mapping to be registered. Synchronization with the port mapping + // goroutine works like this: For each iteration, we allow other goroutines to run and + // also advance the virtual clock by 1 second. Waiting stops when the NAT interface + // has received some requests, or when the clock reaches a timeout. + deadline := clock.Now().Add(portMapRefreshInterval) + for clock.Now() < deadline && mockNAT.mapRequests.Load() < 2 { + time.Sleep(10 * time.Millisecond) + clock.Run(1 * time.Second) + } + + if mockNAT.ipRequests.Load() == 0 { + t.Fatal("external IP was never requested") + } + reqCount := mockNAT.mapRequests.Load() + if reqCount != 2 { + t.Error("wrong request count:", reqCount) + } + enr := srv.LocalNode().Node() + if enr.IP().String() != "192.0.2.0" { + t.Error("wrong IP in ENR:", enr.IP()) + } + if enr.TCP() != 30000 { + t.Error("wrong TCP port in ENR:", enr.TCP()) + } + if enr.UDP() != 30000 { + t.Error("wrong UDP port in ENR:", enr.UDP()) + } +} + +type mockNAT struct { + mappedPort uint16 + mapRequests atomic.Int32 + unmapRequests atomic.Int32 + ipRequests atomic.Int32 +} + +func (m *mockNAT) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { + m.mapRequests.Add(1) + return m.mappedPort, nil +} + +func (m *mockNAT) DeleteMapping(protocol string, extport, intport int) error { + m.unmapRequests.Add(1) + return nil +} + +func (m *mockNAT) ExternalIP() (net.IP, error) { + m.ipRequests.Add(1) + return net.ParseIP("192.0.2.0"), nil +} + +func (m *mockNAT) String() string { + return "mockNAT" +} diff --git a/p2p/server_test.go b/p2p/server_test.go index c8bf4c941c..a0491e984a 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -24,6 +24,8 @@ import ( "math/rand" "net" "reflect" + "strconv" + "strings" "testing" "time" @@ -224,6 +226,14 @@ func TestServerRemovePeerDisconnect(t *testing.T) { srv2.Start() defer srv2.Stop() + s := strings.Split(srv2.ListenAddr, ":") + if len(s) != 2 { + t.Fatal("invalid ListenAddr") + } + if port, err := strconv.Atoi(s[1]); err == nil { + srv2.localnode.Set(enr.TCP(uint16(port))) + } + if !syncAddPeer(srv1, srv2.Self()) { t.Fatal("peer not connected") }