From 8bffa79b1afa187a0a0da262d4a17322c521f997 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 21 Mar 2024 09:22:24 -0400 Subject: [PATCH] transparent proxy: add jobspec support (#20144) Add a transparent proxy block to the existing Connect sidecar service proxy block. This changeset is plumbing required to support transparent proxy configuration on the client. Ref: https://github.com/hashicorp/nomad/issues/10628 --- api/consul.go | 74 +++++++++- command/agent/consul/connect.go | 5 + command/agent/job_endpoint.go | 16 ++ jobspec/parse_service.go | 47 ++++++ jobspec/parse_test.go | 9 ++ .../tg-service-connect-proxy.hcl | 10 ++ nomad/job_endpoint_hook_connect.go | 41 +++++- nomad/job_endpoint_hook_connect_test.go | 63 +++++++- nomad/structs/connect.go | 138 ++++++++++++++++++ nomad/structs/connect_test.go | 63 ++++++++ nomad/structs/diff.go | 56 +++++++ nomad/structs/diff_test.go | 99 +++++++++++++ nomad/structs/services.go | 43 +++++- nomad/structs/services_test.go | 57 ++++++++ 14 files changed, 705 insertions(+), 16 deletions(-) diff --git a/api/consul.go b/api/consul.go index 120f35224a7f..93b2dc62259e 100644 --- a/api/consul.go +++ b/api/consul.go @@ -158,12 +158,17 @@ func (st *SidecarTask) Canonicalize() { // ConsulProxy represents a Consul Connect sidecar proxy jobspec block. type ConsulProxy struct { - LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"` - LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"` - Expose *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"` - ExposeConfig *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead. - Upstreams []*ConsulUpstream `hcl:"upstreams,block"` - Config map[string]interface{} `hcl:"config,block"` + LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"` + LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"` + Expose *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"` + ExposeConfig *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead. + Upstreams []*ConsulUpstream `hcl:"upstreams,block"` + + // TransparentProxy configures the Envoy sidecar to use "transparent + // proxying", which creates IP tables rules inside the network namespace to + // ensure traffic flows thru the Envoy proxy + TransparentProxy *ConsulTransparentProxy `mapstructure:"transparent_proxy" hcl:"transparent_proxy,block"` + Config map[string]interface{} `hcl:"config,block"` } func (cp *ConsulProxy) Canonicalize() { @@ -177,6 +182,8 @@ func (cp *ConsulProxy) Canonicalize() { cp.Upstreams = nil } + cp.TransparentProxy.Canonicalize() + for _, upstream := range cp.Upstreams { upstream.Canonicalize() } @@ -257,6 +264,61 @@ func (cu *ConsulUpstream) Canonicalize() { } } +// ConsulTransparentProxy is used to configure the Envoy sidecar for +// "transparent proxying", which creates IP tables rules inside the network +// namespace to ensure traffic flows thru the Envoy proxy +type ConsulTransparentProxy struct { + // UID of the Envoy proxy. Defaults to the default Envoy proxy container + // image user. + UID string `mapstructure:"uid" hcl:"uid,optional"` + + // OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP + // traffic hitting the PROXY_IN_REDIRECT chain will be redirected here. + // Defaults to 15001. + OutboundPort uint16 `mapstructure:"outbound_port" hcl:"outbound_port,optional"` + + // ExcludeInboundPorts is an additional set of ports will be excluded from + // redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set + // will be added to the ports automatically excluded for the Expose.Port and + // Check.Expose fields. + ExcludeInboundPorts []string `mapstructure:"exclude_inbound_ports" hcl:"exclude_inbound_ports,optional"` + + // ExcludeOutboundPorts is a set of outbound ports that will not be + // redirected to the Envoy proxy, specified as port numbers. + ExcludeOutboundPorts []uint16 `mapstructure:"exclude_outbound_ports" hcl:"exclude_outbound_ports,optional"` + + // ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be + // redirected to the Envoy proxy. + ExcludeOutboundCIDRs []string `mapstructure:"exclude_outbound_cidrs" hcl:"exclude_outbound_cidrs,optional"` + + // ExcludeUIDs is a set of user IDs whose network traffic will not be + // redirected through the Envoy proxy. + ExcludeUIDs []string `mapstructure:"exclude_uids" hcl:"exclude_uids,optional"` + + // NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS + // is false and transparent proxy will direct DNS traffic to Consul DNS if + // available on the client. + NoDNS bool `mapstructure:"no_dns" hcl:"no_dns,optional"` +} + +func (tp *ConsulTransparentProxy) Canonicalize() { + if tp == nil { + return + } + if len(tp.ExcludeInboundPorts) == 0 { + tp.ExcludeInboundPorts = nil + } + if len(tp.ExcludeOutboundCIDRs) == 0 { + tp.ExcludeOutboundCIDRs = nil + } + if len(tp.ExcludeOutboundPorts) == 0 { + tp.ExcludeOutboundPorts = nil + } + if len(tp.ExcludeUIDs) == 0 { + tp.ExcludeUIDs = nil + } +} + type ConsulExposeConfig struct { Paths []*ConsulExposePath `mapstructure:"path" hcl:"path,block"` Path []*ConsulExposePath // Deprecated: only to maintain backwards compatibility. Use Paths instead. diff --git a/command/agent/consul/connect.go b/command/agent/consul/connect.go index 3185838be0ab..e70e66af08a5 100644 --- a/command/agent/consul/connect.go +++ b/command/agent/consul/connect.go @@ -145,8 +145,13 @@ func connectSidecarProxy(info structs.AllocInfo, proxy *structs.ConsulProxy, cPo if err != nil { return nil, err } + mode := api.ProxyModeDefault + if proxy.TransparentProxy != nil { + mode = api.ProxyModeTransparent + } return &api.AgentServiceConnectProxyConfig{ + Mode: mode, LocalServiceAddress: proxy.LocalServiceAddress, LocalServicePort: proxy.LocalServicePort, Config: connectProxyConfig(proxy.Config, cPort, info), diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index ef427a03e9e5..05f1f04a1623 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1897,6 +1897,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul LocalServicePort: in.LocalServicePort, Upstreams: apiUpstreamsToStructs(in.Upstreams), Expose: apiConsulExposeConfigToStructs(expose), + TransparentProxy: apiConnectTransparentProxyToStructs(in.TransparentProxy), Config: maps.Clone(in.Config), } } @@ -1949,6 +1950,21 @@ func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulE } } +func apiConnectTransparentProxyToStructs(in *api.ConsulTransparentProxy) *structs.ConsulTransparentProxy { + if in == nil { + return nil + } + return &structs.ConsulTransparentProxy{ + UID: in.UID, + OutboundPort: in.OutboundPort, + ExcludeInboundPorts: in.ExcludeInboundPorts, + ExcludeOutboundPorts: in.ExcludeOutboundPorts, + ExcludeOutboundCIDRs: in.ExcludeOutboundCIDRs, + ExcludeUIDs: in.ExcludeUIDs, + NoDNS: in.NoDNS, + } +} + func apiConsulExposePathsToStructs(in []*api.ConsulExposePath) []structs.ConsulExposePath { if len(in) == 0 { return nil diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index 6bd443a5ea45..a9e3f2684652 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -927,6 +927,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { "local_service_port", "upstreams", "expose", + "transparent_proxy", "config", } @@ -942,6 +943,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { delete(m, "upstreams") delete(m, "expose") + delete(m, "transparent_proxy") delete(m, "config") dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ @@ -985,6 +987,16 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { } } + if tpo := listVal.Filter("transparent_proxy"); len(tpo.Items) > 1 { + return nil, fmt.Errorf("only 1 transparent_proxy object supported") + } else if len(tpo.Items) == 1 { + if tp, err := parseTproxy(tpo.Items[0]); err != nil { + return nil, err + } else { + proxy.TransparentProxy = tp + } + } + // If we have config, then parse that if o := listVal.Filter("config"); len(o.Items) > 1 { return nil, fmt.Errorf("only 1 meta object supported") @@ -1077,6 +1089,41 @@ func parseExposePath(epo *ast.ObjectItem) (*api.ConsulExposePath, error) { return &path, nil } +func parseTproxy(epo *ast.ObjectItem) (*api.ConsulTransparentProxy, error) { + valid := []string{ + "uid", + "outbound_port", + "exclude_inbound_ports", + "exclude_outbound_ports", + "exclude_outbound_cidrs", + "exclude_uids", + "no_dns", + } + + if err := checkHCLKeys(epo.Val, valid); err != nil { + return nil, multierror.Prefix(err, "tproxy ->") + } + + var tproxy api.ConsulTransparentProxy + var m map[string]interface{} + if err := hcl.DecodeObject(&m, epo.Val); err != nil { + return nil, err + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &tproxy, + }) + if err != nil { + return nil, err + } + + if err := dec.Decode(m); err != nil { + return nil, err + } + + return &tproxy, nil +} + func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { valid := []string{ "destination_name", diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index d9f6774a1906..14d2194af859 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1470,6 +1470,15 @@ func TestParse(t *testing.T) { DestinationName: "upstream2", LocalBindPort: 2002, }}, + TransparentProxy: &api.ConsulTransparentProxy{ + UID: "101", + OutboundPort: 15001, + ExcludeInboundPorts: []string{"www", "9000"}, + ExcludeOutboundPorts: []uint16{443, 80}, + ExcludeOutboundCIDRs: []string{"10.0.0.0/8"}, + ExcludeUIDs: []string{"10", "1001"}, + NoDNS: true, + }, Config: map[string]interface{}{ "foo": "bar", }, diff --git a/jobspec/test-fixtures/tg-service-connect-proxy.hcl b/jobspec/test-fixtures/tg-service-connect-proxy.hcl index 138d58943a79..e09076c52acd 100644 --- a/jobspec/test-fixtures/tg-service-connect-proxy.hcl +++ b/jobspec/test-fixtures/tg-service-connect-proxy.hcl @@ -40,6 +40,16 @@ job "service-connect-proxy" { } } + transparent_proxy { + uid = "101" + outbound_port = 15001 + exclude_inbound_ports = ["www", "9000"] + exclude_outbound_ports = [443, 80] + exclude_outbound_cidrs = ["10.0.0.0/8"] + exclude_uids = ["10", "1001"] + no_dns = true + } + config { foo = "bar" } diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go index 95c0ccc41f31..df0087620e06 100644 --- a/nomad/job_endpoint_hook_connect.go +++ b/nomad/job_endpoint_hook_connect.go @@ -561,33 +561,68 @@ func groupConnectValidate(g *structs.TaskGroup) error { } } - if err := groupConnectUpstreamsValidate(g.Name, g.Services); err != nil { + if err := groupConnectUpstreamsValidate(g, g.Services); err != nil { return err } return nil } -func groupConnectUpstreamsValidate(group string, services []*structs.Service) error { +func groupConnectUpstreamsValidate(g *structs.TaskGroup, services []*structs.Service) error { listeners := make(map[string]string) // address -> service + var connectBlockCount int + var hasTproxy bool + for _, service := range services { + if service.Connect != nil { + connectBlockCount++ + } if service.Connect.HasSidecar() && service.Connect.SidecarService.Proxy != nil { for _, up := range service.Connect.SidecarService.Proxy.Upstreams { listener := net.JoinHostPort(up.LocalBindAddress, strconv.Itoa(up.LocalBindPort)) if s, exists := listeners[listener]; exists { return fmt.Errorf( "Consul Connect services %q and %q in group %q using same address for upstreams (%s)", - service.Name, s, group, listener, + service.Name, s, g.Name, listener, ) } listeners[listener] = service.Name } + + if tp := service.Connect.SidecarService.Proxy.TransparentProxy; tp != nil { + hasTproxy = true + for _, portLabel := range tp.ExcludeInboundPorts { + if !transparentProxyPortLabelValidate(g, portLabel) { + return fmt.Errorf( + "Consul Connect transparent proxy port %q must be numeric or one of network.port labels", portLabel) + } + } + } + } } + if hasTproxy && connectBlockCount > 1 { + return fmt.Errorf("Consul Connect transparent proxy requires there is only one connect block") + } return nil } +func transparentProxyPortLabelValidate(g *structs.TaskGroup, portLabel string) bool { + if _, err := strconv.ParseUint(portLabel, 10, 64); err == nil { + return true + } + + for _, network := range g.Networks { + for _, reservedPort := range network.ReservedPorts { + if reservedPort.Label == portLabel { + return true + } + } + } + return false +} + func groupConnectSidecarValidate(g *structs.TaskGroup, s *structs.Service) error { if n := len(g.Networks); n != 1 { return fmt.Errorf("Consul Connect sidecars require exactly 1 network, found %d in group %q", n, g.Name) diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go index 668031c3bae9..c741f6e7db79 100644 --- a/nomad/job_endpoint_hook_connect_test.go +++ b/nomad/job_endpoint_hook_connect_test.go @@ -548,13 +548,15 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) { ci.Parallel(t) t.Run("no connect services", func(t *testing.T) { - err := groupConnectUpstreamsValidate("group", + tg := &structs.TaskGroup{Name: "group"} + err := groupConnectUpstreamsValidate(tg, []*structs.Service{{Name: "s1"}, {Name: "s2"}}) - require.NoError(t, err) + must.NoError(t, err) }) t.Run("connect services no overlap", func(t *testing.T) { - err := groupConnectUpstreamsValidate("group", + tg := &structs.TaskGroup{Name: "group"} + err := groupConnectUpstreamsValidate(tg, []*structs.Service{ { Name: "s1", @@ -589,11 +591,12 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) { }, }, }) - require.NoError(t, err) + must.NoError(t, err) }) t.Run("connect services overlap port", func(t *testing.T) { - err := groupConnectUpstreamsValidate("group", + tg := &structs.TaskGroup{Name: "group"} + err := groupConnectUpstreamsValidate(tg, []*structs.Service{ { Name: "s1", @@ -628,7 +631,55 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) { }, }, }) - require.EqualError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`) + must.EqError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`) + }) + + t.Run("connect tproxy excludes invalid port", func(t *testing.T) { + tg := &structs.TaskGroup{Name: "group", Networks: structs.Networks{ + { + ReservedPorts: []structs.Port{{ + Label: "www", + }}, + }, + }} + err := groupConnectUpstreamsValidate(tg, + []*structs.Service{ + { + Name: "s1", + Connect: &structs.ConsulConnect{ + SidecarService: &structs.ConsulSidecarService{ + Proxy: &structs.ConsulProxy{ + TransparentProxy: &structs.ConsulTransparentProxy{ + ExcludeInboundPorts: []string{"www", "9000", "no-such-label"}, + }, + }, + }, + }, + }, + }) + must.EqError(t, err, `Consul Connect transparent proxy port "no-such-label" must be numeric or one of network.port labels`) + }) + + t.Run("Consul Connect transparent proxy allows only one Connect block", func(t *testing.T) { + tg := &structs.TaskGroup{Name: "group"} + err := groupConnectUpstreamsValidate(tg, + []*structs.Service{ + { + Name: "s1", + Connect: &structs.ConsulConnect{}, + }, + { + Name: "s2", + Connect: &structs.ConsulConnect{ + SidecarService: &structs.ConsulSidecarService{ + Proxy: &structs.ConsulProxy{ + TransparentProxy: &structs.ConsulTransparentProxy{}, + }, + }, + }, + }, + }) + must.EqError(t, err, `Consul Connect transparent proxy requires there is only one connect block`) }) } diff --git a/nomad/structs/connect.go b/nomad/structs/connect.go index 26e7f30c81ba..34e76875e15b 100644 --- a/nomad/structs/connect.go +++ b/nomad/structs/connect.go @@ -3,6 +3,16 @@ package structs +import ( + "fmt" + "net/netip" + "slices" + "strconv" + + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/helper" +) + // ConsulConfigEntries represents Consul ConfigEntry definitions from a job for // a single Consul namespace. type ConsulConfigEntries struct { @@ -43,3 +53,131 @@ func (j *Job) ConfigEntries() map[string]*ConsulConfigEntries { return collection } + +// ConsulTransparentProxy is used to configure the Envoy sidecar for +// "transparent proxying", which creates IP tables rules inside the network +// namespace to ensure traffic flows thru the Envoy proxy +type ConsulTransparentProxy struct { + + // UID of the Envoy proxy. Defaults to the default Envoy proxy container + // image user. + UID string + + // OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP + // traffic hitting the PROXY_IN_REDIRECT chain will be redirected here. + // Defaults to 15001. + OutboundPort uint16 + + // ExcludeInboundPorts is an additional set of ports will be excluded from + // redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set + // will be added to the ports automatically excluded for the Expose.Port and + // Check.Expose fields. + ExcludeInboundPorts []string + + // ExcludeOutboundPorts is a set of outbound ports that will not be + // redirected to the Envoy proxy, specified as port numbers. + ExcludeOutboundPorts []uint16 + + // ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be + // redirected to the Envoy proxy. + ExcludeOutboundCIDRs []string + + // ExcludeUIDs is a set of user IDs whose network traffic will not be + // redirected through the Envoy proxy. + ExcludeUIDs []string + + // NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS + // is false and transparent proxy will direct DNS traffic to Consul DNS if + // available on the client. + NoDNS bool +} + +func (tp *ConsulTransparentProxy) Copy() *ConsulTransparentProxy { + if tp == nil { + return nil + } + ntp := new(ConsulTransparentProxy) + *ntp = *tp + + ntp.ExcludeInboundPorts = slices.Clone(tp.ExcludeInboundPorts) + ntp.ExcludeOutboundPorts = slices.Clone(tp.ExcludeOutboundPorts) + ntp.ExcludeOutboundCIDRs = slices.Clone(tp.ExcludeOutboundCIDRs) + ntp.ExcludeUIDs = slices.Clone(tp.ExcludeUIDs) + + return ntp +} + +func (tp *ConsulTransparentProxy) Validate() error { + var mErr multierror.Error + + for _, rawCidr := range tp.ExcludeOutboundCIDRs { + _, err := netip.ParsePrefix(rawCidr) + if err != nil { + // note: error returned always include parsed string + mErr.Errors = append(mErr.Errors, + fmt.Errorf("could not parse transparent proxy excluded outbound CIDR as network prefix: %w", err)) + } + } + + requireUIDisUint := func(uidRaw string) error { + _, err := strconv.ParseUint(uidRaw, 10, 16) + if err != nil { + e, ok := err.(*strconv.NumError) + if !ok { + return fmt.Errorf("invalid user ID %q: %w", uidRaw, err) + } + return fmt.Errorf("invalid user ID %q: %w", uidRaw, e.Err) + } + return nil + } + + if tp.UID != "" { + if err := requireUIDisUint(tp.UID); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("transparent proxy block has invalid UID field: %w", err)) + } + } + for _, uid := range tp.ExcludeUIDs { + if err := requireUIDisUint(uid); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("transparent proxy block has invalid ExcludeUIDs field: %w", err)) + } + } + + // note: ExcludeInboundPorts are validated in connect validation hook + // because we need information from the network block + + if mErr.Len() == 1 { + return mErr.Errors[0] + } + return mErr.ErrorOrNil() +} + +func (tp *ConsulTransparentProxy) Equal(o *ConsulTransparentProxy) bool { + if tp == nil || o == nil { + return tp == o + } + if tp.UID != o.UID { + return false + } + if tp.OutboundPort != o.OutboundPort { + return false + } + if !helper.SliceSetEq(tp.ExcludeInboundPorts, o.ExcludeInboundPorts) { + return false + } + if !helper.SliceSetEq(tp.ExcludeOutboundPorts, o.ExcludeOutboundPorts) { + return false + } + if !helper.SliceSetEq(tp.ExcludeOutboundCIDRs, o.ExcludeOutboundCIDRs) { + return false + } + if !helper.SliceSetEq(tp.ExcludeUIDs, o.ExcludeUIDs) { + return false + } + if tp.NoDNS != o.NoDNS { + return false + } + + return true +} diff --git a/nomad/structs/connect_test.go b/nomad/structs/connect_test.go index c7361af89976..ee2403f909b9 100644 --- a/nomad/structs/connect_test.go +++ b/nomad/structs/connect_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/hashicorp/nomad/ci" + "github.com/shoenig/test/must" "github.com/stretchr/testify/require" ) @@ -25,3 +26,65 @@ func TestTaskKind_IsAnyConnectGateway(t *testing.T) { require.False(t, NewTaskKind("", "foo").IsAnyConnectGateway()) }) } + +func TestConnectTransparentProxy_Validate(t *testing.T) { + testCases := []struct { + name string + tp *ConsulTransparentProxy + expectErr string + }{ + { + name: "empty is valid", + tp: &ConsulTransparentProxy{}, + }, + { + name: "invalid CIDR", + tp: &ConsulTransparentProxy{ExcludeOutboundCIDRs: []string{"192.168.1.1"}}, + expectErr: `could not parse transparent proxy excluded outbound CIDR as network prefix: netip.ParsePrefix("192.168.1.1"): no '/'`, + }, + { + name: "invalid UID", + tp: &ConsulTransparentProxy{UID: "foo"}, + expectErr: `transparent proxy block has invalid UID field: invalid user ID "foo": invalid syntax`, + }, + { + name: "invalid ExcludeUIDs", + tp: &ConsulTransparentProxy{ExcludeUIDs: []string{"500000"}}, + expectErr: `transparent proxy block has invalid ExcludeUIDs field: invalid user ID "500000": value out of range`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.tp.Validate() + if tc.expectErr != "" { + must.EqError(t, err, tc.expectErr) + } else { + must.NoError(t, err) + } + }) + } + +} + +func TestConnectTransparentProxy_Equal(t *testing.T) { + tp1 := &ConsulTransparentProxy{ + UID: "101", + OutboundPort: 1001, + ExcludeInboundPorts: []string{"9000", "443"}, + ExcludeOutboundPorts: []uint16{443, 80}, + ExcludeOutboundCIDRs: []string{"10.0.0.0/8", "192.168.1.1"}, + ExcludeUIDs: []string{"1001", "10"}, + NoDNS: true, + } + tp2 := &ConsulTransparentProxy{ + UID: "101", + OutboundPort: 1001, + ExcludeInboundPorts: []string{"443", "9000"}, + ExcludeOutboundPorts: []uint16{80, 443}, + ExcludeOutboundCIDRs: []string{"192.168.1.1", "10.0.0.0/8"}, + ExcludeUIDs: []string{"10", "1001"}, + NoDNS: true, + } + must.Equal(t, tp1, tp2) +} diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index 1b6bbed671f4..60e68b3663f8 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/flatmap" "github.com/mitchellh/hashstructure" ) @@ -1734,6 +1735,10 @@ func consulProxyDiff(old, new *ConsulProxy, contextual bool) *ObjectDiff { diff.Objects = append(diff.Objects, exposeDiff) } + if tproxyDiff := consulTProxyDiff(old.TransparentProxy, new.TransparentProxy, contextual); tproxyDiff != nil { + diff.Objects = append(diff.Objects, tproxyDiff) + } + // diff the config blob if cDiff := configDiff(old.Config, new.Config, contextual); cDiff != nil { diff.Objects = append(diff.Objects, cDiff) @@ -1844,6 +1849,57 @@ func consulProxyExposeDiff(prev, next *ConsulExposeConfig, contextual bool) *Obj return diff } +func consulTProxyDiff(prev, next *ConsulTransparentProxy, contextual bool) *ObjectDiff { + + diff := &ObjectDiff{Type: DiffTypeNone, Name: "TransparentProxy"} + var oldPrimFlat, newPrimFlat map[string]string + + if prev.Equal(next) { + return diff + } else if prev == nil { + prev = &ConsulTransparentProxy{} + diff.Type = DiffTypeAdded + newPrimFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = &ConsulTransparentProxy{} + diff.Type = DiffTypeDeleted + oldPrimFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimFlat = flatmap.Flatten(prev, nil, true) + newPrimFlat = flatmap.Flatten(next, nil, true) + } + + // diff the primitive fields + diff.Fields = fieldDiffs(oldPrimFlat, newPrimFlat, contextual) + + if setDiff := stringSetDiff(prev.ExcludeInboundPorts, next.ExcludeInboundPorts, + "ExcludeInboundPorts", contextual); setDiff != nil && setDiff.Type != DiffTypeNone { + diff.Objects = append(diff.Objects, setDiff) + } + + if setDiff := stringSetDiff( + helper.ConvertSlice(prev.ExcludeOutboundPorts, func(a uint16) string { return fmt.Sprint(a) }), + helper.ConvertSlice(next.ExcludeOutboundPorts, func(a uint16) string { return fmt.Sprint(a) }), + "ExcludeOutboundPorts", + contextual, + ); setDiff != nil && setDiff.Type != DiffTypeNone { + diff.Objects = append(diff.Objects, setDiff) + } + + if setDiff := stringSetDiff(prev.ExcludeOutboundCIDRs, next.ExcludeOutboundCIDRs, + "ExcludeOutboundCIDRs", contextual); setDiff != nil && setDiff.Type != DiffTypeNone { + diff.Objects = append(diff.Objects, setDiff) + } + + if setDiff := stringSetDiff(prev.ExcludeUIDs, next.ExcludeUIDs, + "ExcludeUIDs", contextual); setDiff != nil && setDiff.Type != DiffTypeNone { + diff.Objects = append(diff.Objects, setDiff) + } + + return diff +} + // serviceCheckDiffs diffs a set of service checks. If contextual diff is // enabled, unchanged fields within objects nested in the tasks will be // returned. diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index cef7736881e4..f0106c536947 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -3490,6 +3490,15 @@ func TestTaskGroupDiff(t *testing.T) { Config: map[string]interface{}{ "foo": "qux", }, + TransparentProxy: &ConsulTransparentProxy{ + UID: "101", + OutboundPort: 15001, + ExcludeInboundPorts: []string{"www", "9000"}, + ExcludeOutboundPorts: []uint16{4443}, + ExcludeOutboundCIDRs: []string{"10.0.0.0/8"}, + ExcludeUIDs: []string{"1", "10"}, + NoDNS: true, + }, }, }, Gateway: &ConsulGateway{ @@ -3924,6 +3933,92 @@ func TestTaskGroupDiff(t *testing.T) { }, }, }, + { + Type: DiffTypeAdded, + Name: "TransparentProxy", + Objects: []*ObjectDiff{ + { + Type: DiffTypeAdded, + Name: "ExcludeInboundPorts", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "ExcludeInboundPorts", + Old: "", + New: "9000", + }, + { + Type: DiffTypeAdded, + Name: "ExcludeInboundPorts", + Old: "", + New: "www", + }, + }, + }, + { + Type: DiffTypeAdded, + Name: "ExcludeOutboundPorts", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "ExcludeOutboundPorts", + Old: "", + New: "4443", + }, + }, + }, + { + Type: DiffTypeAdded, + Name: "ExcludeOutboundCIDRs", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "ExcludeOutboundCIDRs", + Old: "", + New: "10.0.0.0/8", + }, + }, + }, + { + Type: DiffTypeAdded, + Name: "ExcludeUIDs", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "ExcludeUIDs", + Old: "", + New: "1", + }, + { + Type: DiffTypeAdded, + Name: "ExcludeUIDs", + Old: "", + New: "10", + }, + }, + }, + }, + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "NoDNS", + Old: "", + New: "true", + }, + { + Type: DiffTypeAdded, + Name: "OutboundPort", + Old: "", + New: "15001", + }, + { + Type: DiffTypeAdded, + Name: "UID", + Old: "", + New: "101", + }, + }, + }, { Type: DiffTypeAdded, Name: "Config", @@ -10024,6 +10119,10 @@ func TestServicesDiff(t *testing.T) { }, Objects: nil, }, + { + Type: DiffTypeNone, + Name: "TransparentProxy", + }, }, }, }, diff --git a/nomad/structs/services.go b/nomad/structs/services.go index 6fc1a3d9e03d..6d5006f762e6 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -958,6 +958,7 @@ func hashConnect(h hash.Hash, connect *ConsulConnect) { hashString(h, p.LocalServiceAddress) hashString(h, strconv.Itoa(p.LocalServicePort)) hashConfig(h, p.Config) + hashTProxy(h, p.TransparentProxy) for _, upstream := range p.Upstreams { hashString(h, upstream.DestinationName) hashString(h, upstream.DestinationNamespace) @@ -1015,6 +1016,22 @@ func hashConfig(h hash.Hash, c map[string]interface{}) { _, _ = fmt.Fprintf(h, "%v", c) } +func hashTProxy(h hash.Hash, tp *ConsulTransparentProxy) { + if tp == nil { + return + } + + hashStringIfNonEmpty(h, tp.UID) + hashIntIfNonZero(h, "OutboundPort", int(tp.OutboundPort)) + hashTags(h, tp.ExcludeInboundPorts) + for _, port := range tp.ExcludeOutboundPorts { + hashIntIfNonZero(h, "ExcludeOutboundPorts", int(port)) + } + hashTags(h, tp.ExcludeOutboundCIDRs) + hashTags(h, tp.ExcludeUIDs) + hashBool(h, tp.NoDNS, "NoDNS") +} + // Equal returns true if the structs are recursively equal. func (s *Service) Equal(o *Service) bool { if s == nil || o == nil { @@ -1187,6 +1204,14 @@ func (c *ConsulConnect) IsMesh() bool { return c.IsGateway() && c.Gateway.Mesh != nil } +// HasTransparentProxy checks if a service with a Connect sidecar has a +// transparent proxy configuration +func (c *ConsulConnect) HasTransparentProxy() bool { + return c.HasSidecar() && + c.SidecarService.Proxy != nil && + c.SidecarService.Proxy.TransparentProxy != nil +} + // Validate that the Connect block represents exactly one of: // - Connect non-native service sidecar proxy // - Connect native service @@ -1201,6 +1226,11 @@ func (c *ConsulConnect) Validate() error { count := 0 if c.HasSidecar() { + if c.HasTransparentProxy() { + if err := c.SidecarService.Proxy.TransparentProxy.Validate(); err != nil { + return err + } + } count++ } @@ -1222,7 +1252,8 @@ func (c *ConsulConnect) Validate() error { } } - // The Native and Sidecar cases are validated up at the service level. + // Checking against the surrounding task group is validated up at the + // service level or job endpint connect validation hook return nil } @@ -1509,6 +1540,11 @@ type ConsulProxy struct { // used by task-group level service checks using HTTP or gRPC protocols. Expose *ConsulExposeConfig + // TransparentProxy configures the Envoy sidecar to use "transparent + // proxying", which creates IP tables rules inside the network namespace to + // ensure traffic flows thru the Envoy proxy + TransparentProxy *ConsulTransparentProxy + // Config is a proxy configuration. It is opaque to Nomad and passed // directly to Consul. Config map[string]interface{} @@ -1525,6 +1561,7 @@ func (p *ConsulProxy) Copy() *ConsulProxy { LocalServicePort: p.LocalServicePort, Expose: p.Expose.Copy(), Upstreams: slices.Clone(p.Upstreams), + TransparentProxy: p.TransparentProxy.Copy(), Config: maps.Clone(p.Config), } } @@ -1551,6 +1588,10 @@ func (p *ConsulProxy) Equal(o *ConsulProxy) bool { return false } + if !p.TransparentProxy.Equal(o.TransparentProxy) { + return false + } + // envoy config, use reflect if !reflect.DeepEqual(p.Config, o.Config) { return false diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index 312b72968026..338261939c35 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -432,6 +432,15 @@ func TestService_Hash(t *testing.T) { LocalBindPort: 29000, Config: map[string]any{"foo": "bar"}, }}, + TransparentProxy: &ConsulTransparentProxy{ + UID: "101", + OutboundPort: 15001, + ExcludeInboundPorts: []string{"www", "9000"}, + ExcludeOutboundPorts: []uint16{4443}, + ExcludeOutboundCIDRs: []string{"10.0.0.0/8"}, + ExcludeUIDs: []string{"1", "10"}, + NoDNS: true, + }, }, Meta: map[string]string{ "test-key": "test-value", @@ -529,6 +538,54 @@ func TestService_Hash(t *testing.T) { t.Run("mod connect sidecar proxy upstream config", func(t *testing.T) { try(t, func(s *svc) { s.Connect.SidecarService.Proxy.Upstreams[0].Config = map[string]any{"foo": "baz"} }) }) + + t.Run("mod connect transparent proxy removed", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy = nil + }) + }) + + t.Run("mod connect transparent proxy uid", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.UID = "42" + }) + }) + + t.Run("mod connect transparent proxy outbound port", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.OutboundPort = 42 + }) + }) + + t.Run("mod connect transparent proxy inbound ports", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeInboundPorts = []string{"443"} + }) + }) + + t.Run("mod connect transparent proxy outbound ports", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeOutboundPorts = []uint16{42} + }) + }) + + t.Run("mod connect transparent proxy outbound cidr", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeOutboundCIDRs = []string{"192.168.1.0/24"} + }) + }) + + t.Run("mod connect transparent proxy exclude uids", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeUIDs = []string{"42"} + }) + }) + + t.Run("mod connect transparent proxy no dns", func(t *testing.T) { + try(t, func(s *svc) { + s.Connect.SidecarService.Proxy.TransparentProxy.NoDNS = false + }) + }) } func TestConsulConnect_Validate(t *testing.T) {