[mobile]Create a real dummy socket option to manage network type hashing (#38099)

Commit Message: Create a real dummy socket option to manage network type
hashing
Additional Description: The previous approach uses IP_TTL and it might
cause unwanted behavior in production.
Risk Level: low
Testing: existing tests
Docs Changes: n/a
Release Notes: n/a
Platform Specific Features: mobile only
Runtime guard: envoy_reloadable_features_use_network_type_socket_option

---------

Signed-off-by: Renjie Tang <renjietang@google.com>

Author: RenjieTang

mobile/library/common/network/BUILD

@@ -14,6 +14,7 @@ envoy_cc_library(
     ],
     repository = "@envoy",
     deps = [
+        ":network_type_socket_option_lib",
         ":proxy_settings_lib",
         "//library/common:engine_types_lib",
         "//library/common/network:src_addr_socket_option_lib",
@@ -54,6 +55,18 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "network_type_socket_option_lib",
+    srcs = ["network_type_socket_option_impl.cc"],
+    hdrs = ["network_type_socket_option_impl.h"],
+    repository = "@envoy",
+    deps = [
+        "@envoy//source/common/common:scalar_to_byte_vector_lib",
+        "@envoy//source/common/network:socket_option_lib",
+        "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
+    ],
+)
+
 envoy_cc_library(
     name = "synthetic_address_lib",
     hdrs = ["synthetic_address_impl.h"],

mobile/library/common/network/connectivity_manager.cc

@@ -13,6 +13,7 @@
 #include "source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h"
 
 #include "fmt/ostream.h"
+#include "library/common/network/network_type_socket_option_impl.h"
 #include "library/common/network/src_addr_socket_option_impl.h"
 
 // Used on Linux (requires root/CAP_NET_RAW)
@@ -308,11 +309,15 @@ Socket::OptionsSharedPtr ConnectivityManagerImpl::getUpstreamSocketOptions(int n
   // Envoy uses the hash signature of overridden socket options to choose a connection pool.
   // Setting a dummy socket option is a hack that allows us to select a different
   // connection pool without materially changing the socket configuration.
-  int ttl_value = DEFAULT_IP_TTL + static_cast<int>(network);
   auto options = std::make_shared<Socket::Options>();
-  options->push_back(std::make_shared<AddrFamilyAwareSocketOptionImpl>(
-      envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_TTL,
-      ENVOY_SOCKET_IPV6_UNICAST_HOPS, ttl_value));
+  if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.use_network_type_socket_option")) {
+    options->push_back(std::make_shared<AddrFamilyAwareSocketOptionImpl>(
+        envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_TTL,
+        ENVOY_SOCKET_IPV6_UNICAST_HOPS, DEFAULT_IP_TTL + static_cast<int>(network)));
+  } else {
+    options->push_back(std::make_shared<NetworkTypeSocketOptionImpl>(network));
+  }
+
   return options;
 }
 

mobile/library/common/network/network_type_socket_option_impl.cc

@@ -0,0 +1,38 @@
+#include "library/common/network/network_type_socket_option_impl.h"
+
+#include "envoy/config/core/v3/base.pb.h"
+#include "envoy/network/socket.h"
+
+#include "source/common/common/scalar_to_byte_vector.h"
+
+#include "absl/types/optional.h"
+
+namespace Envoy {
+namespace Network {
+
+NetworkTypeSocketOptionImpl::NetworkTypeSocketOptionImpl(int network_type)
+    : optname_(0, 0, "network_type"), network_type_(network_type) {}
+
+bool NetworkTypeSocketOptionImpl::setOption(
+    Socket& /*socket*/, envoy::config::core::v3::SocketOption::SocketState /*state*/) const {
+  return true;
+}
+
+void NetworkTypeSocketOptionImpl::hashKey(std::vector<uint8_t>& hash_key) const {
+  pushScalarToByteVector(network_type_, hash_key);
+}
+
+absl::optional<Socket::Option::Details> NetworkTypeSocketOptionImpl::getOptionDetails(
+    const Socket&, envoy::config::core::v3::SocketOption::SocketState /*state*/) const {
+  Socket::Option::Details details;
+  details.name_ = optname_;
+  std::vector<uint8_t> data;
+  pushScalarToByteVector(network_type_, data);
+  details.value_ = std::string(reinterpret_cast<char*>(data.data()), data.size());
+  return absl::make_optional(std::move(details));
+}
+
+bool NetworkTypeSocketOptionImpl::isSupported() const { return true; }
+
+} // namespace Network
+} // namespace Envoy

mobile/library/common/network/network_type_socket_option_impl.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "envoy/config/core/v3/base.pb.h"
+#include "envoy/network/socket.h"
+
+#include "absl/types/optional.h"
+
+namespace Envoy {
+namespace Network {
+
+/**
+ * This is a "dummy" socket option implementation, which does not actually
+ * set any socket option, but rather applies a network type tag to the socket so that requests from
+ * different network types gets hashed to different connections.
+ */
+class NetworkTypeSocketOptionImpl : public Network::Socket::Option {
+public:
+  NetworkTypeSocketOptionImpl(int network_type);
+
+  // Socket::Option
+  bool setOption(Network::Socket& socket,
+                 envoy::config::core::v3::SocketOption::SocketState state) const override;
+  void hashKey(std::vector<uint8_t>& hash_key) const override;
+  absl::optional<Details>
+  getOptionDetails(const Network::Socket& socket,
+                   envoy::config::core::v3::SocketOption::SocketState state) const override;
+  bool isSupported() const override;
+
+private:
+  const Network::SocketOptionName optname_;
+
+  int network_type_;
+};
+
+} // namespace Network
+} // namespace Envoy

mobile/test/common/network/connectivity_manager_test.cc

@@ -242,5 +242,23 @@ TEST_F(ConnectivityManagerTest, IgnoresDuplicatedProxySettingsUpdates) {
   EXPECT_EQ(proxy_settings1, connectivity_manager_->getProxySettings());
 }
 
+TEST_F(ConnectivityManagerTest, NetworkChangeResultsInDifferentSocketOptionsHash) {
+  Runtime::maybeSetRuntimeGuard("envoy.reloadable_features.use_network_type_socket_option", true);
+  auto options1 = std::make_shared<Socket::Options>();
+  connectivity_manager_->addUpstreamSocketOptions(options1);
+  std::vector<uint8_t> hash1;
+  for (const auto& option : *options1) {
+    option->hashKey(hash1);
+  }
+  ConnectivityManagerImpl::setPreferredNetwork(64);
+  auto options2 = std::make_shared<Socket::Options>();
+  connectivity_manager_->addUpstreamSocketOptions(options2);
+  std::vector<uint8_t> hash2;
+  for (const auto& option : *options2) {
+    option->hashKey(hash2);
+  }
+  EXPECT_NE(hash1, hash2);
+}
+
 } // namespace Network
 } // namespace Envoy

source/common/runtime/runtime_features.cc

@@ -167,6 +167,9 @@ FALSE_RUNTIME_GUARD(envoy_reloadable_features_google_grpc_disable_tls_13);
 // before downstream.
 FALSE_RUNTIME_GUARD(envoy_reloadable_features_allow_multiplexed_upstream_half_close);
 
+// TODO(renjietang): Flip to true after prod testing.
+FALSE_RUNTIME_GUARD(envoy_reloadable_features_use_network_type_socket_option);
+
 // Block of non-boolean flags. Use of int flags is deprecated. Do not add more.
 ABSL_FLAG(uint64_t, re2_max_program_size_error_level, 100, ""); // NOLINT
 ABSL_FLAG(uint64_t, re2_max_program_size_warn_level,            // NOLINT