Merge pull request #2225 from subspace/improve-network-connectivity

Improve network connectivity

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs

@@ -193,7 +193,7 @@ struct DsnArgs {
     #[arg(long, default_value_t = 100)]
     pending_out_connections: u32,
     /// Defines target total (in and out) connection number that should be maintained.
-    #[arg(long, default_value_t = 25)]
+    #[arg(long, default_value_t = 15)]
     target_connections: u32,
     /// Known external addresses
     #[arg(long, alias = "external-address")]

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm/dsn.rs

@@ -182,13 +182,19 @@ pub(super) fn configure_dsn(
         max_pending_outgoing_connections: pending_out_connections,
         max_established_incoming_connections: in_connections,
         max_pending_incoming_connections: pending_in_connections,
-        general_target_connections: target_connections,
-        // maintain permanent connections between farmers
-        special_connected_peers_handler: Some(Arc::new(PeerInfo::is_farmer)),
-        // other (non-farmer) connections
+        // Non-farmer connections
         general_connected_peers_handler: Some(Arc::new(|peer_info| {
             !PeerInfo::is_farmer(peer_info)
         })),
+        // Proactively maintain permanent connections with farmers
+        special_connected_peers_handler: Some(Arc::new(PeerInfo::is_farmer)),
+        // Do not have any target for general peers
+        general_connected_peers_target: 0,
+        special_connected_peers_target: target_connections,
+        // Allow up to quarter of incoming connections to be maintained
+        general_connected_peers_limit: in_connections / 4,
+        // Allow to maintain some extra farmer connections beyond direct interest too
+        special_connected_peers_limit: target_connections + in_connections / 4,
         bootstrap_addresses: bootstrap_nodes,
         kademlia_mode: KademliaMode::Dynamic {
             initial_mode: Mode::Client,

crates/subspace-networking/src/bin/subspace-bootstrap-node/main.rs

@@ -163,9 +163,12 @@ async fn main() -> Result<(), Box<dyn Error>> {
                 max_established_outgoing_connections: out_peers,
                 max_pending_incoming_connections: pending_in_peers,
                 max_pending_outgoing_connections: pending_out_peers,
-                // we don't maintain permanent connections with any peer
-                general_connected_peers_handler: None,
-                special_connected_peers_handler: None,
+                // Maintain proactive connections with all peers
+                general_connected_peers_handler: Some(Arc::new(|_| true)),
+                // Maintain some number of persistent connections
+                general_connected_peers_target: in_peers / 8,
+                // Allow some more persistent connections from other peers
+                general_connected_peers_limit: in_peers / 4,
                 bootstrap_addresses: bootstrap_nodes,
                 kademlia_mode: KademliaMode::Static(Mode::Server),
                 external_addresses,

crates/subspace-networking/src/constructor.rs

@@ -59,8 +59,8 @@ const PEER_INFO_PROTOCOL_NAME: &str = "/subspace/peer-info/1.0.0";
 const GENERAL_CONNECTED_PEERS_PROTOCOL_LOG_TARGET: &str = "general-connected-peers";
 const SPECIAL_CONNECTED_PEERS_PROTOCOL_LOG_TARGET: &str = "special-connected-peers";
 
-// Defines max_negotiating_inbound_streams constant for the swarm.
-// It must be set for large plots.
+/// Defines max_negotiating_inbound_streams constant for the swarm.
+/// It must be set for large plots.
 const SWARM_MAX_NEGOTIATING_INBOUND_STREAMS: usize = 100000;
 /// The default maximum established incoming connection number for the swarm.
 const SWARM_MAX_ESTABLISHED_INCOMING_CONNECTIONS: u32 = 100;
@@ -70,12 +70,10 @@ const SWARM_MAX_ESTABLISHED_OUTGOING_CONNECTIONS: u32 = 100;
 const SWARM_MAX_PENDING_INCOMING_CONNECTIONS: u32 = 80;
 /// The default maximum pending incoming connection number for the swarm.
 const SWARM_MAX_PENDING_OUTGOING_CONNECTIONS: u32 = 80;
-// The default maximum connection number to be maintained for the swarm.
-const SWARM_TARGET_CONNECTION_NUMBER: u32 = 25;
 const KADEMLIA_QUERY_TIMEOUT: Duration = Duration::from_secs(40);
 const SWARM_MAX_ESTABLISHED_CONNECTIONS_PER_PEER: Option<u32> = Some(3);
 // TODO: Consider moving this constant to configuration or removing `Toggle` wrapper when we find a
-// use-case for gossipsub protocol.
+//  use-case for gossipsub protocol.
 const ENABLE_GOSSIP_PROTOCOL: bool = false;
 
 const TEMPORARY_BANS_CACHE_SIZE: NonZeroUsize = NonZeroUsize::new(10_000).expect("Not zero; qed");
@@ -236,15 +234,23 @@ pub struct Config<LocalRecordProvider> {
     /// Specifies a source for peer information. None disables the protocol.
     pub peer_info_provider: Option<PeerInfoProvider>,
     /// Defines whether we maintain a persistent connection for common peers.
-    /// None disables the protocol.
+    /// `None` (the default) disables the protocol.
     pub general_connected_peers_handler: Option<ConnectedPeersHandler>,
     /// Defines whether we maintain a persistent connection for special peers.
-    /// None disables the protocol.
+    /// `None` (the default) disables the protocol.
     pub special_connected_peers_handler: Option<ConnectedPeersHandler>,
-    /// Defines target total (in and out) connection number that should be maintained for general peers.
-    pub general_target_connections: u32,
-    /// Defines target total (in and out) connection number that should be maintained for special peers.
-    pub special_target_connections: u32,
+    /// Defines target total (in and out) connection number that should be maintained for general
+    /// peers (defaults to 0).
+    pub general_connected_peers_target: u32,
+    /// Defines target total (in and out) connection number that should be maintained for special
+    /// peers (defaults to 0).
+    pub special_connected_peers_target: u32,
+    /// Defines max total (in and out) connection number that should be maintained for general
+    /// peers (defaults to 0, will be automatically raised if set lower than target).
+    pub general_connected_peers_limit: u32,
+    /// Defines max total (in and out) connection number that should be maintained for special
+    /// peers (defaults to 0, will be automatically raised if set lower than target).
+    pub special_connected_peers_limit: u32,
     /// Addresses to bootstrap Kademlia network
     pub bootstrap_addresses: Vec<Multiaddr>,
     /// Kademlia mode. The default value is set to Static(Client). The peer won't add its address
@@ -367,15 +373,15 @@ where
             metrics: None,
             protocol_version,
             peer_info_provider,
-            // we don't need to keep additional connections by default
+            // Don't need to keep additional connections by default
             general_connected_peers_handler: None,
             special_connected_peers_handler: None,
-            general_target_connections: SWARM_TARGET_CONNECTION_NUMBER,
-            special_target_connections: SWARM_TARGET_CONNECTION_NUMBER,
+            general_connected_peers_target: 0,
+            special_connected_peers_target: 0,
+            general_connected_peers_limit: 0,
+            special_connected_peers_limit: 0,
             bootstrap_addresses: Vec::new(),
-            kademlia_mode: KademliaMode::Dynamic {
-                initial_mode: Mode::Client,
-            },
+            kademlia_mode: KademliaMode::Static(Mode::Client),
             external_addresses: Vec::new(),
             enable_autonat: true,
             disable_bootstrap_on_start: false,
@@ -436,8 +442,10 @@ where
         peer_info_provider,
         general_connected_peers_handler: general_connection_decision_handler,
         special_connected_peers_handler: special_connection_decision_handler,
-        general_target_connections,
-        special_target_connections,
+        general_connected_peers_target,
+        special_connected_peers_target,
+        general_connected_peers_limit,
+        special_connected_peers_limit,
         bootstrap_addresses,
         kademlia_mode,
         external_addresses,
@@ -492,14 +500,18 @@ where
         general_connected_peers_config: general_connection_decision_handler.as_ref().map(|_| {
             ConnectedPeersConfig {
                 log_target: GENERAL_CONNECTED_PEERS_PROTOCOL_LOG_TARGET,
-                target_connected_peers: general_target_connections,
+                target_connected_peers: general_connected_peers_target,
+                max_connected_peers: general_connected_peers_limit
+                    .max(general_connected_peers_target),
                 ..ConnectedPeersConfig::default()
             }
         }),
         special_connected_peers_config: special_connection_decision_handler.as_ref().map(|_| {
             ConnectedPeersConfig {
                 log_target: SPECIAL_CONNECTED_PEERS_PROTOCOL_LOG_TARGET,
-                target_connected_peers: special_target_connections,
+                target_connected_peers: special_connected_peers_target,
+                max_connected_peers: special_connected_peers_limit
+                    .max(special_connected_peers_target),
                 ..ConnectedPeersConfig::default()
             }
         }),

crates/subspace-networking/src/protocols/connected_peers.rs

@@ -107,8 +107,10 @@ pub struct Config {
     pub dialing_interval: Duration,
     /// Interval between logging of the internal state.
     pub logging_interval: Duration,
-    /// Number of connected peers that protocol will maintain.
+    /// Number of connected peers that protocol will maintain proactively.
     pub target_connected_peers: u32,
+    /// Number of connected peers that protocol will allow to be connected to permanently.
+    pub max_connected_peers: u32,
     /// We dial peers using this batch size.
     pub dialing_peer_batch_size: u32,
     /// Time interval reserved for a decision about connections.
@@ -123,7 +125,8 @@ impl Default for Config {
             log_target: DEFAULT_CONNECTED_PEERS_LOG_TARGET,
             dialing_interval: Duration::from_secs(15),
             logging_interval: Duration::from_secs(5),
-            target_connected_peers: 30,
+            target_connected_peers: 15,
+            max_connected_peers: 30,
             dialing_peer_batch_size: 5,
             decision_timeout: Duration::from_secs(10),
         }
@@ -229,7 +232,7 @@ impl<Instance> Behaviour<Instance> {
     /// depend on another protocol (e.g.: PeerInfo protocol event handling).
     pub fn update_keep_alive_status(&mut self, peer_id: PeerId, keep_alive: bool) {
         let not_enough_connected_peers =
-            self.permanently_connected_peers() < self.config.target_connected_peers;
+            self.permanently_connected_peers() < self.config.max_connected_peers;
 
         // It's a known peer.
         if let Some(connection_state) = self.known_peers.get_mut(&peer_id) {
@@ -512,7 +515,7 @@ impl<Instance: 'static + Send> NetworkBehaviour for Behaviour<Instance> {
                 self.dialing_delay.reset(self.config.dialing_interval);
 
                 // Request new peer addresses.
-                if self.peer_cache.is_empty() {
+                if self.peer_cache.is_empty() && self.config.target_connected_peers > 0 {
                     trace!("Requesting new peers for connected-peers protocol...");
 
                     return Poll::Ready(ToSwarm::GenerateEvent(

crates/subspace-networking/src/protocols/connected_peers/tests.rs

@@ -155,45 +155,53 @@ async fn test_new_peer_request() {
         dialing_interval,
         Behaviour::<ConnectedPeersInstance>::new(Config {
             dialing_interval,
+            target_connected_peers: 1,
             ..Default::default()
         }),
     );
 
     peer1.listen().await;
 
-    loop {
-        select! {
-            event = peer1.next_swarm_event().fuse() => {
-                if matches!(event, SwarmEvent::Behaviour(ConnectedPeersEvent::<ConnectedPeersInstance>::NewDialingCandidatesRequested(..))){
-                    break;
-                }
-            },
-            _ = sleep(LONG_DELAY).fuse() => {
-                panic!("No new peers requests.");
-            }
+    let waiting_for_event_fut = async {
+        while !matches!(
+            peer1.next_swarm_event().await,
+            SwarmEvent::Behaviour(
+                ConnectedPeersEvent::<ConnectedPeersInstance>::NewDialingCandidatesRequested(..)
+            )
+        ) {
+            // Wait for interesting event
+        }
+    };
+
+    select! {
+        _ = waiting_for_event_fut.fuse() => {},
+        _ = sleep(LONG_DELAY).fuse() => {
+            panic!("No new peers requests");
         }
     }
 
-    // We've received the new peers request when we don't have enough connected peers.
+    // We've received the new peers request when peer cache is empty
 }
 
 #[tokio::test()]
 async fn test_target_connected_peer_limit_number() {
-    let target_connected_peers = 1;
+    let max_connected_peers = 1;
 
     let mut peer1 = new_ephemeral(
         DECISION_TIMEOUT,
         Behaviour::<ConnectedPeersInstance>::new(Config {
+            target_connected_peers: 0,
+            max_connected_peers,
             decision_timeout: DECISION_TIMEOUT,
-            target_connected_peers,
             ..Default::default()
         }),
     );
 
     let mut peer2 = new_ephemeral(
         DECISION_TIMEOUT,
         Behaviour::<ConnectedPeersInstance>::new(Config {
-            target_connected_peers,
+            target_connected_peers: 0,
+            max_connected_peers,
             decision_timeout: DECISION_TIMEOUT,
             ..Default::default()
         }),
@@ -202,7 +210,8 @@ async fn test_target_connected_peer_limit_number() {
     let mut peer3 = new_ephemeral(
         DECISION_TIMEOUT,
         Behaviour::<ConnectedPeersInstance>::new(Config {
-            target_connected_peers,
+            target_connected_peers: 0,
+            max_connected_peers,
             decision_timeout: DECISION_TIMEOUT,
             ..Default::default()
         }),

crates/subspace-node/src/lib.rs

@@ -259,7 +259,7 @@ pub struct Cli {
     pub dsn_pending_out_connections: u32,
 
     /// Defines target total (in and out) connection number for DSN that should be maintained.
-    #[arg(long, default_value_t = 25)]
+    #[arg(long, default_value_t = 15)]
     pub dsn_target_connections: u32,
 
     /// Determines whether we allow keeping non-global (private, shared, loopback..) addresses

crates/subspace-service/src/dsn.rs

@@ -179,16 +179,16 @@ where
         max_established_outgoing_connections: dsn_config.max_out_connections,
         max_pending_incoming_connections: dsn_config.max_pending_in_connections,
         max_pending_outgoing_connections: dsn_config.max_pending_out_connections,
-        general_target_connections: dsn_config.target_connections,
-        special_target_connections: 0,
-        reserved_peers: dsn_config.reserved_peers,
-        // maintain permanent connections with any peer
+        // Maintain proactive connections with all peers
         general_connected_peers_handler: Some(Arc::new(|_| true)),
+        general_connected_peers_target: dsn_config.target_connections,
+        // Allow to maintain some extra general connections beyond direct interest too
+        general_connected_peers_limit: dsn_config.target_connections
+            + dsn_config.max_in_connections / 4,
+        reserved_peers: dsn_config.reserved_peers,
         bootstrap_addresses: dsn_config.bootstrap_nodes,
         external_addresses: dsn_config.external_addresses,
-        kademlia_mode: KademliaMode::Dynamic {
-            initial_mode: Mode::Client,
-        },
+        kademlia_mode: KademliaMode::Static(Mode::Client),
         metrics,
         disable_bootstrap_on_start: dsn_config.disable_bootstrap_on_start,