Add support for distributing requests over multiple NW interfaces beh…

…ind feature flag (#943)

* Add initial plumbing for multiple NIC support

Signed-off-by: Daniel Carl Jones <djonesoa@amazon.com>

* Update based on feedback

Signed-off-by: Daniel Carl Jones <djonesoa@amazon.com>

* Replace comma-delimited network interfaces arg with bind arg

Signed-off-by: Daniel Carl Jones <djonesoa@amazon.com>

---------

Signed-off-by: Daniel Carl Jones <djonesoa@amazon.com>

mountpoint-s3-client/src/s3_crt_client.rs

@@ -92,6 +92,7 @@ pub struct S3ClientConfig {
     max_attempts: Option<NonZeroUsize>,
     read_backpressure: bool,
     initial_read_window: usize,
+    network_interface_names: Vec<String>,
 }
 
 impl Default for S3ClientConfig {
@@ -109,6 +110,7 @@ impl Default for S3ClientConfig {
             max_attempts: None,
             read_backpressure: false,
             initial_read_window: DEFAULT_PART_SIZE,
+            network_interface_names: vec![],
         }
     }
 }
@@ -203,6 +205,13 @@ impl S3ClientConfig {
         self.initial_read_window = initial_read_window;
         self
     }
+
+    /// Set a list of network interfaces to distribute S3 requests over
+    #[must_use = "S3ClientConfig follows a builder pattern"]
+    pub fn network_interface_names(mut self, network_interface_names: Vec<String>) -> Self {
+        self.network_interface_names = network_interface_names;
+        self
+    }
 }
 
 /// Authentication configuration for the CRT-based S3 client
@@ -380,6 +389,10 @@ impl S3CrtClientInner {
             }
         }
 
+        if !config.network_interface_names.is_empty() {
+            client_config.network_interface_names(config.network_interface_names);
+        }
+
         let user_agent = config.user_agent.unwrap_or_else(|| UserAgent::new(None));
         let user_agent_header = user_agent.build();
 

mountpoint-s3-crt/src/s3/client.rs

@@ -61,6 +61,56 @@ pub struct ClientConfig {
 
     /// The region
     region: Option<String>,
+
+    /// List of network interfaces to be used by the client.
+    ///
+    /// The client will determine the logic for balancing requests over the network interfaces
+    /// according to its own logic.
+    network_interface_names: Option<NetworkInterfaceNames>,
+}
+
+/// This struct bundles together the list of owned strings for the network interfaces, and the
+/// cursors pointing to them.
+///
+/// This allows [ClientConfig] to keep the strings backing the cursors allocated until at least
+/// the time of [Client::new] (where the content of the cursors is copied),
+/// and deallocate when [ClientConfig] is dropped.
+#[derive(Debug)]
+struct NetworkInterfaceNames {
+    /// The list of network interface names.
+    ///
+    /// We use a boxed array of strings, as we have no need for a mutable list like [Vec].
+    /// The [String] entries must never be mutated, the cursors point to their underlying memory.
+    _names: Box<[String]>,
+
+    /// List of owned cursors. This will be pointed at by [ClientConfig]'s inner [aws_s3_client_config].
+    cursors: Box<[aws_byte_cursor]>,
+}
+
+impl NetworkInterfaceNames {
+    /// Create a new [NetworkInterfaceNamesInner].
+    pub fn new(names: Vec<String>) -> Self {
+        // Turn this into a read-only representation of the vector of strings.
+        let names = names.into_boxed_slice();
+
+        let cursors = names
+            .iter()
+            .map(|name| {
+                // SAFETY: The names are stored alongside the cursors in NetworkInterfaceNamesInner.
+                //         The lifetime of NetworkInterfaceNamesInner will always meet the lifetime of the cursors.
+                //         We never mutate the String backing these byte cursors once created.
+                unsafe { name.as_aws_byte_cursor() }
+            })
+            .collect::<Vec<_>>()
+            .into_boxed_slice();
+
+        Self { _names: names, cursors }
+    }
+
+    /// Immutable slice of interface names as [aws_byte_cursor].
+    pub fn aws_byte_cursors(&self) -> &[aws_byte_cursor] {
+        self.cursors.as_ref()
+    }
 }
 
 impl ClientConfig {
@@ -84,6 +134,19 @@ impl ClientConfig {
         self
     }
 
+    /// Replace list of network interface names to be used by S3 clients created with this config.
+    pub fn network_interface_names(&mut self, network_interface_names: Vec<String>) -> &mut Self {
+        let network_interface_names = self
+            .network_interface_names
+            .insert(NetworkInterfaceNames::new(network_interface_names));
+
+        // The cursor array will always outlive the inner config.
+        self.inner.network_interface_names_array = network_interface_names.aws_byte_cursors().as_ptr();
+        self.inner.num_network_interface_names = network_interface_names.aws_byte_cursors().len();
+
+        self
+    }
+
     /// Retry strategy used to reschedule failed requests
     pub fn retry_strategy(&mut self, retry_strategy: RetryStrategy) -> &mut Self {
         self.inner.retry_strategy = retry_strategy.inner.as_ptr();

mountpoint-s3/Cargo.toml

@@ -80,6 +80,7 @@ built = { version = "0.7.1", features = ["git2"] }
 [features]
 # Unreleased feature flags
 event_log = []
+multiple-nw-iface = []
 # Features for choosing tests
 fips_tests = []
 fuse_tests = []

mountpoint-s3/src/cli.rs

@@ -314,6 +314,15 @@ pub struct CliArgs {
         value_name = "ALGORITHM",
     )]
     pub upload_checksums: Option<UploadChecksums>,
+
+    #[cfg(feature = "multiple-nw-iface")]
+    #[clap(
+        long,
+        help = "One or more network interfaces for Mountpoint to use when accessing S3. Requires Linux 5.7+ or running as root.",
+        help_heading = CLIENT_OPTIONS_HEADER,
+        value_name = "NETWORK_INTERFACE",
+    )]
+    pub bind: Option<Vec<String>>,
 }
 
 #[derive(Debug, Clone)]
@@ -635,13 +644,21 @@ pub fn create_s3_client(args: &CliArgs) -> anyhow::Result<(S3CrtClient, EventLoo
     if let Some(ttl) = args.metadata_ttl {
         user_agent.key_value("mp-cache-ttl", &ttl.to_string());
     }
+    #[cfg(feature = "multiple-nw-iface")]
+    if let Some(interfaces) = &args.bind {
+        user_agent.key_value("mp-nw-interfaces", &interfaces.len().to_string());
+    }
 
     let mut client_config = S3ClientConfig::new()
         .auth_config(auth_config)
         .throughput_target_gbps(throughput_target_gbps)
         .read_part_size(args.read_part_size.unwrap_or(args.part_size) as usize)
         .write_part_size(args.write_part_size.unwrap_or(args.part_size) as usize)
         .user_agent(user_agent);
+    #[cfg(feature = "multiple-nw-iface")]
+    if let Some(interfaces) = &args.bind {
+        client_config = client_config.network_interface_names(interfaces.clone());
+    }
     if args.requester_pays {
         client_config = client_config.request_payer("requester");
     }