Improve Lighthouse Connectivity Via ENR TCP Update (#4057)

Currently Lighthouse will remain uncontactable if users port forward a port that is not the same as the one they are listening on. 

For example, if Lighthouse runs with port 9000 TCP/UDP locally but a router is configured to pass 9010 externally to the lighthouse node on 9000, other nodes on the network will not be able to reach the lighthouse node. 

This occurs because Lighthouse does not update its ENR TCP port on external socket discovery. The intention was always that users should use `--enr-tcp-port` to customise this, but this is non-intuitive. 

The difficulty arises because we have no discovery mechanism to find our external TCP port. If we discovery a new external UDP port, we must guess what our external TCP port might be. This PR assumes the external TCP port is the same as the external UDP port (which may not be the case) and thus updates the TCP port along with the UDP port if the `--enr-tcp-port` flag is not set. 

Along with this PR, will be added documentation to the Lighthouse book so users can correctly understand and configure their ENR to maximize Lighthouse's connectivity. 

This relies on https://github.com/sigp/discv5/pull/166 and we should wait for a new release in discv5 before adding this PR.

Cargo.lock

@@ -1605,16 +1605,6 @@ version = "0.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b72465f46d518f6015d9cf07f7f3013a95dd6b9c2747c3d65ae0cce43929d14f"
 
-[[package]]
-name = "delay_map"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c4d75d3abfe4830dcbf9bcb1b926954e121669f74dd1ca7aa0183b1755d83f6"
-dependencies = [
- "futures",
- "tokio-util 0.6.10",
-]
-
 [[package]]
 name = "delay_map"
 version = "0.3.0"
@@ -1816,15 +1806,15 @@ dependencies = [
 
 [[package]]
 name = "discv5"
-version = "0.1.0"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767c0e59b3e8d65222d95df723cc2ea1da92bb0f27c563607e6f0bde064f255"
+checksum = "b009a99b85b58900df46435307fc5c4c845af7e182582b1fbf869572fa9fce69"
 dependencies = [
  "aes 0.7.5",
  "aes-gcm 0.9.4",
  "arrayvec",
- "delay_map 0.1.2",
+ "delay_map",
- "enr",
+ "enr 0.7.0",
  "fnv",
  "futures",
  "hashlink 0.7.0",
@@ -1973,6 +1963,25 @@ name = "enr"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26fa0a0be8915790626d5759eb51fe47435a8eac92c2f212bd2da9aa7f30ea56"
+dependencies = [
+ "base64 0.13.1",
+ "bs58",
+ "bytes",
+ "hex",
+ "k256",
+ "log",
+ "rand 0.8.5",
+ "rlp",
+ "serde",
+ "sha3 0.10.6",
+ "zeroize",
+]
+
+[[package]]
+name = "enr"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "492a7e5fc2504d5fdce8e124d3e263b244a68b283cac67a69eda0cd43e0aebad"
 dependencies = [
  "base64 0.13.1",
  "bs58",
@@ -2221,7 +2230,7 @@ dependencies = [
 name = "eth2_network_config"
 version = "0.2.0"
 dependencies = [
- "enr",
+ "discv5",
  "eth2_config",
  "eth2_ssz",
  "serde_yaml",
@@ -2372,7 +2381,7 @@ dependencies = [
  "async-stream",
  "blst",
  "bs58",
- "enr",
+ "enr 0.6.2",
  "hex",
  "integer-sqrt",
  "multiaddr 0.14.0",
@@ -4415,7 +4424,7 @@ dependencies = [
 name = "lighthouse_network"
 version = "0.2.0"
 dependencies = [
- "delay_map 0.3.0",
+ "delay_map",
  "directory",
  "dirs",
  "discv5",
@@ -5032,7 +5041,7 @@ name = "network"
 version = "0.2.0"
 dependencies = [
  "beacon_chain",
- "delay_map 0.3.0",
+ "delay_map",
  "derivative",
  "environment",
  "error-chain",

beacon_node/lighthouse_network/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2021"
 
 [dependencies]
-discv5 = { version = "0.1.0", features = ["libp2p"] }
+discv5 = { version = "0.2.2", features = ["libp2p"] }
 unsigned-varint = { version = "0.6.0", features = ["codec"] }
 types = { path =  "../../consensus/types" }
 eth2_ssz_types = "0.2.2"

beacon_node/lighthouse_network/src/discovery/mod.rs

@@ -177,6 +177,13 @@ pub struct Discovery<TSpec: EthSpec> {
     /// always false.
     pub started: bool,
 
+    /// This keeps track of whether an external UDP port change should also indicate an internal
+    /// TCP port change. As we cannot detect our external TCP port, we assume that the external UDP
+    /// port is also our external TCP port. This assumption only holds if the user has not
+    /// explicitly set their ENR TCP port via the CLI config. The first indicates tcp4 and the
+    /// second indicates tcp6.
+    update_tcp_port: (bool, bool),
+
     /// Logger for the discovery behaviour.
     log: slog::Logger,
 }
@@ -300,6 +307,11 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             }
         }
 
+        let update_tcp_port = (
+            config.enr_tcp4_port.is_none(),
+            config.enr_tcp6_port.is_none(),
+        );
+
         Ok(Self {
             cached_enrs: LruCache::new(50),
             network_globals,
@@ -309,6 +321,7 @@ impl<TSpec: EthSpec> Discovery<TSpec> {
             discv5,
             event_stream,
             started: !config.disable_discovery,
+            update_tcp_port,
             log,
             enr_dir,
         })
@@ -1019,6 +1032,13 @@ impl<TSpec: EthSpec> NetworkBehaviour for Discovery<TSpec> {
                             metrics::check_nat();
                             // Discv5 will have updated our local ENR. We save the updated version
                             // to disk.
+
+                            if (self.update_tcp_port.0 && socket_addr.is_ipv4())
+                                || (self.update_tcp_port.1 && socket_addr.is_ipv6())
+                            {
+                                // Update the TCP port in the ENR
+                                self.discv5.update_local_enr_socket(socket_addr, true);
+                            }
                             let enr = self.discv5.local_enr();
                             enr::save_enr_to_disk(Path::new(&self.enr_dir), &enr, &self.log);
                             // update  network globals

beacon_node/lighthouse_network/src/metrics.rs

@@ -167,7 +167,8 @@ pub fn check_nat() {
 }
 
 pub fn scrape_discovery_metrics() {
-    let metrics = discv5::metrics::Metrics::from(discv5::Discv5::raw_metrics());
+    let metrics =
+        discv5::metrics::Metrics::from(discv5::Discv5::<discv5::DefaultProtocolId>::raw_metrics());
     set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second);
     set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64);
     set_gauge(&DISCOVERY_SENT_BYTES, metrics.bytes_sent as i64);

book/src/advanced_networking.md

@@ -41,7 +41,7 @@ drastically and use the (recommended) default.
 
 ### NAT Traversal (Port Forwarding)
 
-Lighthouse, by default, used port 9000 for both TCP and UDP. Lighthouse will
+Lighthouse, by default, uses port 9000 for both TCP and UDP. Lighthouse will
 still function if it is behind a NAT without any port mappings. Although
 Lighthouse still functions, we recommend that some mechanism is used to ensure
 that your Lighthouse node is publicly accessible. This will typically improve
@@ -54,6 +54,16 @@ node will inform you of established routes in this case). If UPnP is not
 enabled, we recommend you manually set up port mappings to both of Lighthouse's
 TCP and UDP ports (9000 by default).
 
+> Note: Lighthouse needs to advertise its publicly accessible ports in
+> order to inform its peers that it is contactable and how to connect to it. 
+> Lighthouse has an automated way of doing this for the UDP port. This means
+> Lighthouse can detect its external UDP port. There is no such mechanism for the
+> TCP port. As such, we assume that the external UDP and external TCP port is the
+> same (i.e external 5050 UDP/TCP mapping to internal 9000 is fine). If you are setting up differing external UDP and TCP ports, you should
+> explicitly specify them using the `--enr-tcp-port` and `--enr-udp-port` as
+> explained in the following section.
+
+
 ### ENR Configuration
 
 Lighthouse has a number of CLI parameters for constructing and modifying the

book/src/faq.md

@@ -128,8 +128,9 @@ same `datadir` as a previous network. I.e if you have been running the
 `datadir` (the `datadir` is also printed out in the beacon node's logs on
 boot-up).
 
-If you find yourself with a low peer count and is not reaching the target you
+If you find yourself with a low peer count and it's not reaching the target you
-expect. Try setting up the correct port forwards as described [here](./advanced_networking.md#nat-traversal-port-forwarding).
+expect. Try setting up the correct port forwards as described
+[here](./advanced_networking.md#nat-traversal-port-forwarding).
 
 ### What should I do if I lose my slashing protection database?
 

boot_node/src/server.rs

@@ -44,7 +44,7 @@ pub async fn run<T: EthSpec>(config: BootNodeConfig<T>, log: slog::Logger) {
     info!(log, "Contact information"; "multiaddrs" => ?local_enr.multiaddr_p2p());
 
     // construct the discv5 server
-    let mut discv5 = Discv5::new(local_enr.clone(), local_key, discv5_config).unwrap();
+    let mut discv5: Discv5 = Discv5::new(local_enr.clone(), local_key, discv5_config).unwrap();
 
     // If there are any bootnodes add them to the routing table
     for enr in boot_nodes {

common/eth2_network_config/Cargo.toml

@@ -18,4 +18,4 @@ serde_yaml = "0.8.13"
 types = { path = "../../consensus/types"}
 eth2_ssz = "0.4.1"
 eth2_config = { path = "../eth2_config"}
-enr = { version = "0.6.2", features = ["ed25519", "k256"] }
+discv5 = "0.2.2"

common/eth2_network_config/src/lib.rs

@@ -11,7 +11,7 @@
 //! To add a new built-in testnet, add it to the `define_hardcoded_nets` invocation in the `eth2_config`
 //! crate.
 
-use enr::{CombinedKey, Enr};
+use discv5::enr::{CombinedKey, Enr};
 use eth2_config::{instantiate_hardcoded_nets, HardcodedNet};
 use std::fs::{create_dir_all, File};
 use std::io::{Read, Write};