transparent proxy: add jobspec support (#20144)

Add a transparent proxy block to the existing Connect sidecar service proxy
block. This changeset is plumbing required to support transparent proxy
configuration on the client.

Ref: #10628

api/consul.go

@@ -157,12 +157,17 @@ func (st *SidecarTask) Canonicalize() {
 
 // ConsulProxy represents a Consul Connect sidecar proxy jobspec block.
 type ConsulProxy struct {
-	LocalServiceAddress string                 `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
-	LocalServicePort    int                    `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
-	Expose              *ConsulExposeConfig    `mapstructure:"expose" hcl:"expose,block"`
-	ExposeConfig        *ConsulExposeConfig    // Deprecated: only to maintain backwards compatibility. Use Expose instead.
-	Upstreams           []*ConsulUpstream      `hcl:"upstreams,block"`
-	Config              map[string]interface{} `hcl:"config,block"`
+	LocalServiceAddress string              `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
+	LocalServicePort    int                 `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
+	Expose              *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
+	ExposeConfig        *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead.
+	Upstreams           []*ConsulUpstream   `hcl:"upstreams,block"`
+
+	// TransparentProxy configures the Envoy sidecar to use "transparent
+	// proxying", which creates IP tables rules inside the network namespace to
+	// ensure traffic flows thru the Envoy proxy
+	TransparentProxy *ConsulTransparentProxy `mapstructure:"transparent_proxy" hcl:"transparent_proxy,block"`
+	Config           map[string]interface{}  `hcl:"config,block"`
 }
 
 func (cp *ConsulProxy) Canonicalize() {
@@ -176,6 +181,8 @@ func (cp *ConsulProxy) Canonicalize() {
 		cp.Upstreams = nil
 	}
 
+	cp.TransparentProxy.Canonicalize()
+
 	for _, upstream := range cp.Upstreams {
 		upstream.Canonicalize()
 	}
@@ -263,6 +270,61 @@ func (cu *ConsulUpstream) Canonicalize() {
 	}
 }
 
+// ConsulTransparentProxy is used to configure the Envoy sidecar for
+// "transparent proxying", which creates IP tables rules inside the network
+// namespace to ensure traffic flows thru the Envoy proxy
+type ConsulTransparentProxy struct {
+	// UID of the Envoy proxy. Defaults to the default Envoy proxy container
+	// image user.
+	UID string `mapstructure:"uid" hcl:"uid,optional"`
+
+	// OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP
+	// traffic hitting the PROXY_IN_REDIRECT chain will be redirected here.
+	// Defaults to 15001.
+	OutboundPort uint16 `mapstructure:"outbound_port" hcl:"outbound_port,optional"`
+
+	// ExcludeInboundPorts is an additional set of ports will be excluded from
+	// redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set
+	// will be added to the ports automatically excluded for the Expose.Port and
+	// Check.Expose fields.
+	ExcludeInboundPorts []string `mapstructure:"exclude_inbound_ports" hcl:"exclude_inbound_ports,optional"`
+
+	// ExcludeOutboundPorts is a set of outbound ports that will not be
+	// redirected to the Envoy proxy, specified as port numbers.
+	ExcludeOutboundPorts []uint16 `mapstructure:"exclude_outbound_ports" hcl:"exclude_outbound_ports,optional"`
+
+	// ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be
+	// redirected to the Envoy proxy.
+	ExcludeOutboundCIDRs []string `mapstructure:"exclude_outbound_cidrs" hcl:"exclude_outbound_cidrs,optional"`
+
+	// ExcludeUIDs is a set of user IDs whose network traffic will not be
+	// redirected through the Envoy proxy.
+	ExcludeUIDs []string `mapstructure:"exclude_uids" hcl:"exclude_uids,optional"`
+
+	// NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS
+	// is false and transparent proxy will direct DNS traffic to Consul DNS if
+	// available on the client.
+	NoDNS bool `mapstructure:"no_dns" hcl:"no_dns,optional"`
+}
+
+func (tp *ConsulTransparentProxy) Canonicalize() {
+	if tp == nil {
+		return
+	}
+	if len(tp.ExcludeInboundPorts) == 0 {
+		tp.ExcludeInboundPorts = nil
+	}
+	if len(tp.ExcludeOutboundCIDRs) == 0 {
+		tp.ExcludeOutboundCIDRs = nil
+	}
+	if len(tp.ExcludeOutboundPorts) == 0 {
+		tp.ExcludeOutboundPorts = nil
+	}
+	if len(tp.ExcludeUIDs) == 0 {
+		tp.ExcludeUIDs = nil
+	}
+}
+
 type ConsulExposeConfig struct {
 	Paths []*ConsulExposePath `mapstructure:"path" hcl:"path,block"`
 	Path  []*ConsulExposePath // Deprecated: only to maintain backwards compatibility. Use Paths instead.

command/agent/consul/connect.go

@@ -145,8 +145,13 @@ func connectSidecarProxy(info structs.AllocInfo, proxy *structs.ConsulProxy, cPo
 	if err != nil {
 		return nil, err
 	}
+	mode := api.ProxyModeDefault
+	if proxy.TransparentProxy != nil {
+		mode = api.ProxyModeTransparent
+	}
 
 	return &api.AgentServiceConnectProxyConfig{
+		Mode:                mode,
 		LocalServiceAddress: proxy.LocalServiceAddress,
 		LocalServicePort:    proxy.LocalServicePort,
 		Config:              connectProxyConfig(proxy.Config, cPort, info),

command/agent/job_endpoint.go

@@ -1867,6 +1867,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul
 		LocalServicePort:    in.LocalServicePort,
 		Upstreams:           apiUpstreamsToStructs(in.Upstreams),
 		Expose:              apiConsulExposeConfigToStructs(expose),
+		TransparentProxy:    apiConnectTransparentProxyToStructs(in.TransparentProxy),
 		Config:              maps.Clone(in.Config),
 	}
 }
@@ -1918,6 +1919,21 @@ func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulE
 	}
 }
 
+func apiConnectTransparentProxyToStructs(in *api.ConsulTransparentProxy) *structs.ConsulTransparentProxy {
+	if in == nil {
+		return nil
+	}
+	return &structs.ConsulTransparentProxy{
+		UID:                  in.UID,
+		OutboundPort:         in.OutboundPort,
+		ExcludeInboundPorts:  in.ExcludeInboundPorts,
+		ExcludeOutboundPorts: in.ExcludeOutboundPorts,
+		ExcludeOutboundCIDRs: in.ExcludeOutboundCIDRs,
+		ExcludeUIDs:          in.ExcludeUIDs,
+		NoDNS:                in.NoDNS,
+	}
+}
+
 func apiConsulExposePathsToStructs(in []*api.ConsulExposePath) []structs.ConsulExposePath {
 	if len(in) == 0 {
 		return nil

jobspec/parse_service.go

@@ -769,6 +769,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
 		"local_service_port",
 		"upstreams",
 		"expose",
+		"transparent_proxy",
 		"config",
 	}
 
@@ -784,6 +785,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
 
 	delete(m, "upstreams")
 	delete(m, "expose")
+	delete(m, "transparent_proxy")
 	delete(m, "config")
 
 	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -827,6 +829,16 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
 		}
 	}
 
+	if tpo := listVal.Filter("transparent_proxy"); len(tpo.Items) > 1 {
+		return nil, fmt.Errorf("only 1 transparent_proxy object supported")
+	} else if len(tpo.Items) == 1 {
+		if tp, err := parseTproxy(tpo.Items[0]); err != nil {
+			return nil, err
+		} else {
+			proxy.TransparentProxy = tp
+		}
+	}
+
 	// If we have config, then parse that
 	if o := listVal.Filter("config"); len(o.Items) > 1 {
 		return nil, fmt.Errorf("only 1 meta object supported")
@@ -919,6 +931,41 @@ func parseExposePath(epo *ast.ObjectItem) (*api.ConsulExposePath, error) {
 	return &path, nil
 }
 
+func parseTproxy(epo *ast.ObjectItem) (*api.ConsulTransparentProxy, error) {
+	valid := []string{
+		"uid",
+		"outbound_port",
+		"exclude_inbound_ports",
+		"exclude_outbound_ports",
+		"exclude_outbound_cidrs",
+		"exclude_uids",
+		"no_dns",
+	}
+
+	if err := checkHCLKeys(epo.Val, valid); err != nil {
+		return nil, multierror.Prefix(err, "tproxy ->")
+	}
+
+	var tproxy api.ConsulTransparentProxy
+	var m map[string]interface{}
+	if err := hcl.DecodeObject(&m, epo.Val); err != nil {
+		return nil, err
+	}
+
+	dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		Result: &tproxy,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := dec.Decode(m); err != nil {
+		return nil, err
+	}
+
+	return &tproxy, nil
+}
+
 func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) {
 	valid := []string{
 		"destination_name",

jobspec/parse_test.go

@@ -1468,6 +1468,15 @@ func TestParse(t *testing.T) {
 										DestinationName: "upstream2",
 										LocalBindPort:   2002,
 									}},
+									TransparentProxy: &api.ConsulTransparentProxy{
+										UID:                  "101",
+										OutboundPort:         15001,
+										ExcludeInboundPorts:  []string{"www", "9000"},
+										ExcludeOutboundPorts: []uint16{443, 80},
+										ExcludeOutboundCIDRs: []string{"10.0.0.0/8"},
+										ExcludeUIDs:          []string{"10", "1001"},
+										NoDNS:                true,
+									},
 									Config: map[string]interface{}{
 										"foo": "bar",
 									},

jobspec/test-fixtures/tg-service-connect-proxy.hcl

@@ -40,6 +40,16 @@ job "service-connect-proxy" {
               }
             }
 
+            transparent_proxy {
+              uid                    = "101"
+              outbound_port          = 15001
+              exclude_inbound_ports  = ["www", "9000"]
+              exclude_outbound_ports = [443, 80]
+              exclude_outbound_cidrs = ["10.0.0.0/8"]
+              exclude_uids           = ["10", "1001"]
+              no_dns                 = true
+            }
+
             config {
               foo = "bar"
             }

nomad/job_endpoint_hook_connect.go

@@ -561,33 +561,68 @@ func groupConnectValidate(g *structs.TaskGroup) error {
 		}
 	}
 
-	if err := groupConnectUpstreamsValidate(g.Name, g.Services); err != nil {
+	if err := groupConnectUpstreamsValidate(g, g.Services); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func groupConnectUpstreamsValidate(group string, services []*structs.Service) error {
+func groupConnectUpstreamsValidate(g *structs.TaskGroup, services []*structs.Service) error {
 	listeners := make(map[string]string) // address -> service
 
+	var connectBlockCount int
+	var hasTproxy bool
+
 	for _, service := range services {
+		if service.Connect != nil {
+			connectBlockCount++
+		}
 		if service.Connect.HasSidecar() && service.Connect.SidecarService.Proxy != nil {
 			for _, up := range service.Connect.SidecarService.Proxy.Upstreams {
 				listener := net.JoinHostPort(up.LocalBindAddress, strconv.Itoa(up.LocalBindPort))
 				if s, exists := listeners[listener]; exists {
 					return fmt.Errorf(
 						"Consul Connect services %q and %q in group %q using same address for upstreams (%s)",
-						service.Name, s, group, listener,
+						service.Name, s, g.Name, listener,
 					)
 				}
 				listeners[listener] = service.Name
 			}
+
+			if tp := service.Connect.SidecarService.Proxy.TransparentProxy; tp != nil {
+				hasTproxy = true
+				for _, portLabel := range tp.ExcludeInboundPorts {
+					if !transparentProxyPortLabelValidate(g, portLabel) {
+						return fmt.Errorf(
+							"Consul Connect transparent proxy port %q must be numeric or one of network.port labels", portLabel)
+					}
+				}
+			}
+
 		}
 	}
+	if hasTproxy && connectBlockCount > 1 {
+		return fmt.Errorf("Consul Connect transparent proxy requires there is only one connect block")
+	}
 	return nil
 }
 
+func transparentProxyPortLabelValidate(g *structs.TaskGroup, portLabel string) bool {
+	if _, err := strconv.ParseUint(portLabel, 10, 64); err == nil {
+		return true
+	}
+
+	for _, network := range g.Networks {
+		for _, reservedPort := range network.ReservedPorts {
+			if reservedPort.Label == portLabel {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func groupConnectSidecarValidate(g *structs.TaskGroup, s *structs.Service) error {
 	if n := len(g.Networks); n != 1 {
 		return fmt.Errorf("Consul Connect sidecars require exactly 1 network, found %d in group %q", n, g.Name)