diff --git a/Development.md b/Development.md index e22cbc58d..7eb7b3729 100644 --- a/Development.md +++ b/Development.md @@ -54,7 +54,7 @@ INPOD_UDS=/tmp/ztunnel cargo run --example inpodserver -- pod1 run ztunnel (as root) with: ```shell -RUST_LOG=debug INPOD_ENABLED=true INPOD_UDS=/tmp/ztunnel FAKE_CA="true" XDS_ADDRESS="" LOCAL_XDS_PATH=./examples/localhost.yaml cargo run --features testing +RUST_LOG=debug PROXY_MODE=shared INPOD_UDS=/tmp/ztunnel FAKE_CA="true" XDS_ADDRESS="" LOCAL_XDS_PATH=./examples/localhost.yaml cargo run --features testing ``` (note: to run ztunnel as root, consider using `export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="sudo -E"` so cargo `sudo` the binary) @@ -162,7 +162,7 @@ xargs env <, Receiver>, )> { - let mut manager = setup_netns_test!(TestMode::InPod); + let mut manager = setup_netns_test!(TestMode::Shared); let (server, mut manager) = run_async_blocking(async move { if ztunnel_mode != WorkloadMode::Direct { // we need a client ztunnel diff --git a/src/app.rs b/src/app.rs index 46633bf8b..f19c9ca8f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -134,8 +134,8 @@ pub async fn build_with_cert( ) .map_err(|e| anyhow::anyhow!("failed to start proxy factory {:?}", e))?; - if config.inpod_enabled { - tracing::info!("in-pod mode enabled"); + if config.proxy_mode == config::ProxyMode::Shared { + tracing::info!("shared proxy mode - in-pod mode enabled"); let run_future = init_inpod_proxy_mgr( &mut registry, &mut admin_server, diff --git a/src/config.rs b/src/config.rs index b7c96ada4..33239442e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -37,7 +37,6 @@ const KUBERNETES_SERVICE_HOST: &str = "KUBERNETES_SERVICE_HOST"; const NETWORK: &str = "NETWORK"; const NODE_NAME: &str = "NODE_NAME"; const PROXY_MODE: &str = "PROXY_MODE"; -const INPOD_ENABLED: &str = "INPOD_ENABLED"; const INPOD_MARK: &str = "INPOD_MARK"; const INPOD_UDS: &str = "INPOD_UDS"; const INPOD_PORT_REUSE: &str = "INPOD_PORT_REUSE"; @@ -224,7 +223,6 @@ pub struct Config { // System dns resolver opts used for on-demand ztunnel dns resolution pub dns_resolver_opts: ResolverOpts, - pub inpod_enabled: bool, pub inpod_uds: PathBuf, pub inpod_port_reuse: bool, pub inpod_mark: u32, @@ -490,7 +488,6 @@ pub fn construct_config(pc: ProxyConfig) -> Result { proxy_args: parse_args(), dns_resolver_cfg, dns_resolver_opts, - inpod_enabled: parse_default(INPOD_ENABLED, false)?, inpod_uds: parse_default(INPOD_UDS, PathBuf::from("/var/run/ztunnel/ztunnel.sock"))?, inpod_port_reuse: parse_default(INPOD_PORT_REUSE, true)?, inpod_mark: parse_default(INPOD_MARK, DEFAULT_INPOD_MARK)?, diff --git a/src/proxy/inbound.rs b/src/proxy/inbound.rs index d51224227..c9914f41a 100644 --- a/src/proxy/inbound.rs +++ b/src/proxy/inbound.rs @@ -40,6 +40,7 @@ use crate::state::workload::address::Address; use crate::state::workload::application_tunnel::Protocol as AppProtocol; use crate::{assertions, copy, proxy, socket, strng, tls}; +use crate::config::ProxyMode; use crate::drain::run_with_drain; use crate::proxy::h2; use crate::state::workload::{self, NetworkAddress, Workload}; @@ -187,7 +188,7 @@ impl Inbound { return req.send_error(build_response(StatusCode::BAD_REQUEST)); } }; - let illegal_call = if pi.cfg.inpod_enabled { + let illegal_call = if pi.cfg.proxy_mode == ProxyMode::Shared { // User sent a request to pod:15006. This would forward to pod:15006 infinitely // Use hbone_addr instead of upstream_addr to allow for sandwich mode, which intentionally // sends to 15008. diff --git a/src/proxy/inbound_passthrough.rs b/src/proxy/inbound_passthrough.rs index f5f8144b3..1ae0fd400 100644 --- a/src/proxy/inbound_passthrough.rs +++ b/src/proxy/inbound_passthrough.rs @@ -127,12 +127,14 @@ impl InboundPassthrough { let dest_addr = socket::orig_dst_addr_or_default(&inbound_stream); // Check if it is an illegal call to ourself, which could trampoline to illegal addresses or // lead to infinite loops - let illegal_call = if pi.cfg.inpod_enabled { + let illegal_call = if pi.cfg.proxy_mode == ProxyMode::Shared { // User sent a request to pod:15006. This would forward to pod:15006 infinitely + // OR + // User sent a request to the ztunnel directly. This isn't allowed pi.cfg.illegal_ports.contains(&dest_addr.port()) + || Some(dest_addr.ip()) == pi.cfg.local_ip } else { - // User sent a request to the ztunnel directly. This isn't allowed - pi.cfg.proxy_mode == ProxyMode::Shared && Some(dest_addr.ip()) == pi.cfg.local_ip + false }; if illegal_call { metrics::log_early_deny( diff --git a/src/proxy/outbound.rs b/src/proxy/outbound.rs index d265d7c73..aff415a33 100644 --- a/src/proxy/outbound.rs +++ b/src/proxy/outbound.rs @@ -60,13 +60,13 @@ impl Outbound { transparent, "listener established", ); - let inpod = pi.cfg.inpod_enabled; + let mode = pi.cfg.proxy_mode; Ok(Outbound { pi, listener, drain, // Do not need to spoof with inpod mode for outbound - enable_orig_src: transparent && !inpod, + enable_orig_src: transparent && mode != ProxyMode::Shared, }) } @@ -165,9 +165,9 @@ impl OutboundConnection { // Block calls to ztunnel directly, unless we are in "in-pod". // For in-pod, this isn't an issue and is useful: this allows things like prometheus scraping ztunnel. - if self.pi.cfg.proxy_mode == ProxyMode::Shared + // TODO is this actually wrong for ProxyMode::Dedicated? Seems like a valid case + if self.pi.cfg.proxy_mode != ProxyMode::Shared && Some(dest_addr.ip()) == self.pi.cfg.local_ip - && !self.pi.cfg.inpod_enabled { metrics::log_early_deny(source_addr, dest_addr, Reporter::source, Error::SelfCall); return; @@ -268,7 +268,7 @@ impl OutboundConnection { ) -> Result<(), Error> { // Create a TCP connection to upstream // We do not need spoofing for inbound - let local = if self.enable_orig_src && !self.pi.cfg.inpod_enabled { + let local = if self.enable_orig_src && self.pi.cfg.proxy_mode != ProxyMode::Shared { super::get_original_src_from_stream(&stream) } else { None diff --git a/src/proxy/socks5.rs b/src/proxy/socks5.rs index 5e0f210b0..918050a04 100644 --- a/src/proxy/socks5.rs +++ b/src/proxy/socks5.rs @@ -31,6 +31,7 @@ use tokio::net::TcpStream; use tokio::sync::watch; use tracing::{debug, error, info, info_span, Instrument}; +use crate::config; use crate::drain::run_with_drain; use crate::drain::DrainWatcher; use crate::proxy::outbound::OutboundConnection; @@ -60,13 +61,13 @@ impl Socks5 { "listener established", ); - let inpod = pi.cfg.inpod_enabled; + let mode = pi.cfg.proxy_mode; Ok(Socks5 { pi, listener, drain, // Do not need to spoof with inpod mode for outbound - enable_orig_src: transparent && !inpod, + enable_orig_src: transparent && mode != config::ProxyMode::Shared, }) } diff --git a/src/test_helpers.rs b/src/test_helpers.rs index 9737748e9..7e3dc5c4e 100644 --- a/src/test_helpers.rs +++ b/src/test_helpers.rs @@ -122,6 +122,7 @@ pub fn test_config_with_port_xds_addr_and_root_cert( outbound_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), inbound_plaintext_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), dns_proxy_addr: config::Address::Localhost(false, 0), + proxy_mode: config::ProxyMode::Dedicated, illegal_ports: HashSet::new(), // for "direct" tests, since the ports are latebound, we can't test illegal ports fake_self_inbound: true, // for "direct" tests, since the ports are latebound, we have to do this. Yes, this is test concerns leaking into prod code ..config::parse_config().unwrap() diff --git a/src/test_helpers/linux.rs b/src/test_helpers/linux.rs index 6057a0b1b..54acb20e1 100644 --- a/src/test_helpers/linux.rs +++ b/src/test_helpers/linux.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::config::ConfigSource; +use crate::config::{ConfigSource, ProxyMode}; use crate::rbac::Authorization; use crate::state::service::{endpoint_uid, Endpoint, Service}; use crate::state::workload::{gatewayaddress, Workload}; @@ -24,7 +24,7 @@ use crate::{config, identity, proxy, strng}; use crate::signal::ShutdownTrigger; use crate::test_helpers::inpod::start_ztunnel_server; -use crate::test_helpers::linux::TestMode::InPod; +use crate::test_helpers::linux::TestMode::Shared; use itertools::Itertools; use nix::unistd::mkdtemp; use std::net::IpAddr; @@ -79,8 +79,8 @@ impl Drop for WorkloadManager { #[derive(Clone, Copy, Ord, PartialOrd, PartialEq, Eq)] pub enum TestMode { - InPod, - SharedNode, + Shared, + Dedicated, } impl WorkloadManager { @@ -107,7 +107,7 @@ impl WorkloadManager { /// deploy_ztunnel is called. As such, you must ensure this is called after all other workloads are created. pub async fn deploy_ztunnel(&mut self, node: &str) -> anyhow::Result { let mut inpod_uds: PathBuf = "/dev/null".into(); - let ztunnel_server = if self.mode == InPod { + let ztunnel_server = if self.mode == Shared { inpod_uds = self.tmp_dir.join(node); Some(start_ztunnel_server(inpod_uds.clone())) } else { @@ -125,7 +125,11 @@ impl WorkloadManager { policies: self.policies.clone(), services: self.services.values().cloned().collect_vec(), }; - let inpod_enabled = ztunnel_server.is_some(); + let proxy_mode = if ztunnel_server.is_some() { + ProxyMode::Shared + } else { + ProxyMode::Dedicated + }; let (mut tx_cfg, rx_cfg) = mpsc_ack(1); tx_cfg.send(initial_config).await?; let local_xds_config = Some(ConfigSource::Dynamic(Arc::new(Mutex::new(rx_cfg)))); @@ -137,15 +141,15 @@ impl WorkloadManager { local_node: Some(node.to_string()), local_ip: Some(ns.ip()), inpod_uds, - inpod_enabled, + proxy_mode, ..config::parse_config().unwrap() }; let (tx, rx) = std::sync::mpsc::sync_channel(0); // Setup the ztunnel... let cloned_ns = ns.clone(); ns.run_ready(move |ready| async move { - if !inpod_enabled { - // not needed in inpod mode. In in pod mode we run `ztunnel-redirect-inpod.sh` + if proxy_mode != ProxyMode::Shared { + // not needed in "inpod" (shared proxy) mode. In shared mode we run `ztunnel-redirect-inpod.sh` // inside the pod's netns helpers::run_command(&format!("scripts/ztunnel-redirect.sh {ip}"))?; } @@ -473,7 +477,7 @@ impl<'a> TestWorkloadBuilder<'a> { if self.captured { // Setup redirection let zt_info = self.manager.ztunnels.get_mut(node.as_str()).unwrap(); - if self.manager.mode == InPod { + if self.manager.mode == Shared { // In the new pod network network_namespace .netns() diff --git a/tests/namespaced.rs b/tests/namespaced.rs index 7e7c8c21f..3f780fc04 100644 --- a/tests/namespaced.rs +++ b/tests/namespaced.rs @@ -44,7 +44,7 @@ mod namespaced { use crate::namespaced::WorkloadMode::Captured; use ztunnel::setup_netns_test; - use ztunnel::test_helpers::linux::TestMode::{InPod, SharedNode}; + use ztunnel::test_helpers::linux::TestMode::{Dedicated, Shared}; use ztunnel::test_helpers::linux::WorkloadManager; use ztunnel::test_helpers::netns::{Namespace, Resolver}; use ztunnel::test_helpers::*; @@ -58,7 +58,7 @@ mod namespaced { #[tokio::test] async fn local_captured_inpod() -> anyhow::Result<()> { simple_client_server_test( - setup_netns_test!(InPod), + setup_netns_test!(Shared), Captured(DEFAULT_NODE), Captured(DEFAULT_NODE), ) @@ -67,20 +67,28 @@ mod namespaced { #[tokio::test] async fn server_uncaptured_inpod() -> anyhow::Result<()> { - simple_client_server_test(setup_netns_test!(InPod), Captured(DEFAULT_NODE), Uncaptured) - .await + simple_client_server_test( + setup_netns_test!(Shared), + Captured(DEFAULT_NODE), + Uncaptured, + ) + .await } #[tokio::test] async fn client_uncaptured_inpod() -> anyhow::Result<()> { - simple_client_server_test(setup_netns_test!(InPod), Captured(DEFAULT_NODE), Uncaptured) - .await + simple_client_server_test( + setup_netns_test!(Shared), + Captured(DEFAULT_NODE), + Uncaptured, + ) + .await } #[tokio::test] async fn cross_node_captured_inpod() -> anyhow::Result<()> { simple_client_server_test( - setup_netns_test!(InPod), + setup_netns_test!(Shared), Captured(DEFAULT_NODE), Captured(REMOTE_NODE), ) @@ -91,9 +99,9 @@ mod namespaced { // This is not currently supported since https://github.com/istio/ztunnel/commit/12d154cceb1d20eb1f11ae43c2310e66e93c7120 #[tokio::test] - async fn server_uncaptured_sharednode() -> anyhow::Result<()> { + async fn server_uncaptured_dedicated() -> anyhow::Result<()> { simple_client_server_test( - setup_netns_test!(SharedNode), + setup_netns_test!(Dedicated), Captured(DEFAULT_NODE), Uncaptured, ) @@ -101,9 +109,9 @@ mod namespaced { } #[tokio::test] - async fn client_uncaptured_sharednode() -> anyhow::Result<()> { + async fn client_uncaptured_dedicated() -> anyhow::Result<()> { simple_client_server_test( - setup_netns_test!(SharedNode), + setup_netns_test!(Dedicated), Captured(DEFAULT_NODE), Uncaptured, ) @@ -111,9 +119,9 @@ mod namespaced { } #[tokio::test] - async fn cross_node_captured_sharednode() -> anyhow::Result<()> { + async fn cross_node_captured_dedicated() -> anyhow::Result<()> { simple_client_server_test( - setup_netns_test!(SharedNode), + setup_netns_test!(Dedicated), Captured(DEFAULT_NODE), Captured(REMOTE_NODE), ) @@ -122,7 +130,7 @@ mod namespaced { #[tokio::test] async fn workload_waypoint() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; @@ -180,7 +188,7 @@ mod namespaced { #[tokio::test] async fn service_waypoint() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; @@ -243,7 +251,7 @@ mod namespaced { #[tokio::test] async fn sandwich_waypoint_plain() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let _zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; @@ -279,7 +287,7 @@ mod namespaced { #[tokio::test] async fn sandwich_waypoint_proxy_protocol() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let _zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; @@ -331,7 +339,7 @@ mod namespaced { #[tokio::test] async fn service_loadbalancing() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let local = manager.deploy_ztunnel(DEFAULT_NODE).await?; let remote = manager.deploy_ztunnel(REMOTE_NODE).await?; manager @@ -475,7 +483,7 @@ mod namespaced { #[tokio::test] async fn test_ztunnel_shutdown() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let local = manager.deploy_ztunnel(DEFAULT_NODE).await?; let server = manager .workload_builder("server", DEFAULT_NODE) @@ -521,7 +529,7 @@ mod namespaced { #[tokio::test] async fn test_server_shutdown() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); manager.deploy_ztunnel(DEFAULT_NODE).await?; let server = manager .workload_builder("server", DEFAULT_NODE) @@ -605,7 +613,7 @@ mod namespaced { #[tokio::test] async fn test_policy() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; manager .add_policy(Authorization { @@ -677,7 +685,7 @@ mod namespaced { #[tokio::test] async fn hbone_ip_mismatch() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let zt = manager.deploy_ztunnel(DEFAULT_NODE).await?; let _server = manager .workload_builder("server", DEFAULT_NODE) @@ -736,7 +744,7 @@ mod namespaced { #[tokio::test] async fn malicious_calls_inpod() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let _ztunnel = manager.deploy_ztunnel(DEFAULT_NODE).await?; let client = manager .workload_builder("client", DEFAULT_NODE) @@ -798,34 +806,9 @@ mod namespaced { .await } - #[tokio::test] - async fn malicious_calls_sharednode() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(SharedNode); - let _ztunnel = manager.deploy_ztunnel(DEFAULT_NODE).await?; - let client = manager - .workload_builder("client", DEFAULT_NODE) - .register() - .await?; - - let zt = manager.resolve("ztunnel-node")?; - malicious_calls_test( - client, - vec![ - (zt, 15001, Request), // Outbound: should be blocked due to recursive call - (zt, 15006, Request), // Inbound: should be blocked due to recursive call - (zt, 15008, Request), // HBONE: expected TLS, reject - (zt, 15080, Connection), // Socks5: only localhost - (zt, 15000, Connection), // Admin: only localhost - (zt, 15020, Http), // Stats: accept connection and returns a HTTP error - (zt, 15021, Http), // Readiness: accept connection and returns a HTTP error - ], - ) - .await - } - #[tokio::test] async fn trust_domain_mismatch_rejected() -> anyhow::Result<()> { - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let id = identity::Identity::Spiffe { trust_domain: "clusterset.local".into(), // change to mismatched trustdomain service_account: "my-app".into(), @@ -872,7 +855,7 @@ mod namespaced { #[tokio::test] async fn test_prefetch_forget_certs() -> anyhow::Result<()> { // TODO: this test doesn't really need namespacing, but the direct test doesn't allow dynamic config changes. - let mut manager = setup_netns_test!(InPod); + let mut manager = setup_netns_test!(Shared); let id1 = identity::Identity::Spiffe { trust_domain: "cluster.local".into(), service_account: "sa1".into(),