From af6c2fd361d683789a79e1bd2999605acecfb9c4 Mon Sep 17 00:00:00 2001 From: ZongRun <1181591811hzr@gmail.com> Date: Mon, 21 Aug 2023 15:05:27 +0800 Subject: [PATCH] feat: support expression route for TCPRoute. (#4385) * feat: support expression route for tcproute. * tests: add unittest. * refactor: rename source file and function. * bugfix: clear destinations field. * tests: enable expression flavor of tcproute. * tests: fix tests. * style: change assert style. * style: remove unused const. * docs: update CHANGELOG. --- CHANGELOG.md | 7 + internal/dataplane/parser/atc/field.go | 3 +- .../parser/translate_failures_test.go | 18 -- .../dataplane/parser/translate_tcproute.go | 9 +- .../parser/translate_tcproute_test.go | 261 ++++++++++++++++++ internal/dataplane/parser/translate_utils.go | 10 + .../parser/translators/l4route_atc.go | 27 ++ test/integration/tcproute_test.go | 2 - 8 files changed, 311 insertions(+), 26 deletions(-) create mode 100644 internal/dataplane/parser/translate_tcproute_test.go create mode 100644 internal/dataplane/parser/translators/l4route_atc.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fd30975fa2..f3122175d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,13 @@ Adding a new version? You'll need three changes: ## Unreleased +### Added + +- Added translator to translate `TCPRoute` in gateway APIs to + expression based kong routes. Similar to ingresses, this translator is only + enabled when feature gate `ExpressionRoutes` is turned on and the managed + Kong gateway runs in router flavor `expressions`. + [#4385](https://github.com/Kong/kubernetes-ingress-controller/pull/4385) ### Changes diff --git a/internal/dataplane/parser/atc/field.go b/internal/dataplane/parser/atc/field.go index b48db3380e..afe1008cd8 100644 --- a/internal/dataplane/parser/atc/field.go +++ b/internal/dataplane/parser/atc/field.go @@ -64,7 +64,8 @@ func (f IntField) String() string { // https://docs.konghq.com/gateway/latest/reference/router-expressions-language/#available-fields const ( - FieldNetPort IntField = "net.port" + FieldNetPort IntField = "net.port" + FieldNetDstPort IntField = "net.dst.port" ) // HTTPHeaderField extracts the value of an HTTP header from the request. diff --git a/internal/dataplane/parser/translate_failures_test.go b/internal/dataplane/parser/translate_failures_test.go index 0e1706175c..56e8b6cecb 100644 --- a/internal/dataplane/parser/translate_failures_test.go +++ b/internal/dataplane/parser/translate_failures_test.go @@ -106,18 +106,6 @@ func TestTranslationFailureUnsupportedObjectsExpressionRoutes(t *testing.T) { { name: "TCPRoutes, UDPRoutes and TLSRoutes in gateway APIs are not supported", objects: store.FakeObjects{ - TCPRoutes: []*gatewayv1alpha2.TCPRoute{ - { - TypeMeta: metav1.TypeMeta{ - Kind: "TCPRoute", - APIVersion: gatewayv1alpha2.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "tcproute-1", - Namespace: "default", - }, - }, - }, UDPRoutes: []*gatewayv1alpha2.UDPRoute{ { TypeMeta: metav1.TypeMeta{ @@ -144,12 +132,6 @@ func TestTranslationFailureUnsupportedObjectsExpressionRoutes(t *testing.T) { }, }, causingObjects: []client.Object{ - &gatewayv1alpha2.TCPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tcproute-1", - Namespace: "default", - }, - }, &gatewayv1alpha2.UDPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "udproute-1", diff --git a/internal/dataplane/parser/translate_tcproute.go b/internal/dataplane/parser/translate_tcproute.go index cabec220fb..311d7947c1 100644 --- a/internal/dataplane/parser/translate_tcproute.go +++ b/internal/dataplane/parser/translate_tcproute.go @@ -25,11 +25,6 @@ func (p *Parser) ingressRulesFromTCPRoutes() ingressRules { var errs []error for _, tcproute := range tcpRouteList { - if p.featureFlags.ExpressionRoutes { - p.registerResourceFailureNotSupportedForExpressionRoutes(tcproute) - continue - } - if err := p.ingressRulesFromTCPRoute(&result, tcproute); err != nil { err = fmt.Errorf("TCPRoute %s/%s can't be routed: %w", tcproute.Namespace, tcproute.Name, err) errs = append(errs, err) @@ -40,6 +35,10 @@ func (p *Parser) ingressRulesFromTCPRoutes() ingressRules { } } + if p.featureFlags.ExpressionRoutes { + applyExpressionToIngressRules(&result) + } + if len(errs) > 0 { for _, err := range errs { p.logger.Errorf(err.Error()) diff --git a/internal/dataplane/parser/translate_tcproute_test.go b/internal/dataplane/parser/translate_tcproute_test.go new file mode 100644 index 0000000000..e7cddd1183 --- /dev/null +++ b/internal/dataplane/parser/translate_tcproute_test.go @@ -0,0 +1,261 @@ +package parser + +import ( + "strings" + "testing" + + "github.com/kong/go-kong/kong" + "github.com/samber/lo" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/failures" + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v2/internal/store" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util/builder" +) + +func TestIngressRulesFromTCPRoutesUsingExpressionRoutes(t *testing.T) { + tcpRouteTypeMeta := metav1.TypeMeta{Kind: "TCPRoute", APIVersion: gatewayv1alpha2.SchemeGroupVersion.String()} + + testCases := []struct { + name string + tcpRoutes []*gatewayv1alpha2.TCPRoute + expectedKongServices []kongstate.Service + expectedKongRoutes map[string][]kongstate.Route + expectedFailures []failures.ResourceFailure + }{ + { + name: "tcproute with single rule and single backendref", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`net.dst.port == 80`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + { + name: "tcproute with single rule and multiple backendrefs", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + builder.NewBackendRef("service2").WithPort(443).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + { + Name: "service2", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(443)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`(net.dst.port == 80) || (net.dst.port == 443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + { + name: "tcproute with multiple rules", + tcpRoutes: []*gatewayv1alpha2.TCPRoute{ + { + TypeMeta: tcpRouteTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "tcproute-1", + }, + Spec: gatewayv1alpha2.TCPRouteSpec{ + Rules: []gatewayv1alpha2.TCPRouteRule{ + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service1").WithPort(80).Build(), + builder.NewBackendRef("service2").WithPort(443).Build(), + }, + }, + { + BackendRefs: []gatewayv1alpha2.BackendRef{ + builder.NewBackendRef("service3").WithPort(8080).Build(), + builder.NewBackendRef("service4").WithPort(8443).Build(), + }, + }, + }, + }, + }, + }, + expectedKongServices: []kongstate.Service{ + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.0"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service1", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(80)}, + }, + { + Name: "service2", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(443)}, + }, + }, + }, + { + Service: kong.Service{ + Name: kong.String("tcproute.default.tcproute-1.1"), + }, + Backends: []kongstate.ServiceBackend{ + { + Name: "service3", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(8080)}, + }, + { + Name: "service4", + PortDef: kongstate.PortDef{Mode: kongstate.PortModeByNumber, Number: int32(8443)}, + }, + }, + }, + }, + expectedKongRoutes: map[string][]kongstate.Route{ + "tcproute.default.tcproute-1.0": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.0.0"), + Expression: kong.String(`(net.dst.port == 80) || (net.dst.port == 443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + "tcproute.default.tcproute-1.1": { + { + Route: kong.Route{ + Name: kong.String("tcproute.default.tcproute-1.1.0"), + Expression: kong.String(`(net.dst.port == 8080) || (net.dst.port == 8443)`), + PreserveHost: kong.Bool(true), + Protocols: kong.StringSlice("tcp"), + }, + ExpressionRoutes: true, + }, + }, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + fakestore, err := store.NewFakeStore(store.FakeObjects{TCPRoutes: tc.tcpRoutes}) + require.NoError(t, err) + parser := mustNewParser(t, fakestore) + parser.featureFlags.ExpressionRoutes = true + + failureCollector, err := failures.NewResourceFailuresCollector(logrus.New()) + require.NoError(t, err) + parser.failuresCollector = failureCollector + + result := parser.ingressRulesFromTCPRoutes() + // check services + require.Len(t, result.ServiceNameToServices, len(tc.expectedKongServices), + "should have expected number of services") + for _, expectedKongService := range tc.expectedKongServices { + kongService, ok := result.ServiceNameToServices[*expectedKongService.Name] + require.Truef(t, ok, "should find service %s", expectedKongService.Name) + require.Equal(t, expectedKongService.Backends, kongService.Backends) + // check routes + expectedKongRoutes := tc.expectedKongRoutes[*kongService.Name] + require.Len(t, kongService.Routes, len(expectedKongRoutes)) + + kongRouteNameToRoute := lo.SliceToMap(kongService.Routes, func(r kongstate.Route) (string, kongstate.Route) { + return *r.Name, r + }) + for _, expectedRoute := range expectedKongRoutes { + routeName := expectedRoute.Name + r, ok := kongRouteNameToRoute[*routeName] + require.Truef(t, ok, "should find route %s", *routeName) + require.Equal(t, expectedRoute.Expression, r.Expression) + require.Equal(t, expectedRoute.Protocols, r.Protocols) + } + } + // check translation failures + translationFailures := failureCollector.PopResourceFailures() + require.Len(t, translationFailures, len(tc.expectedFailures)) + for _, expectedTranslationFailure := range tc.expectedFailures { + expectedFailureMessage := expectedTranslationFailure.Message() + require.True(t, lo.ContainsBy(translationFailures, func(failure failures.ResourceFailure) bool { + return strings.Contains(failure.Message(), expectedFailureMessage) + })) + } + }) + } +} diff --git a/internal/dataplane/parser/translate_utils.go b/internal/dataplane/parser/translate_utils.go index ca7e864347..9dd311debf 100644 --- a/internal/dataplane/parser/translate_utils.go +++ b/internal/dataplane/parser/translate_utils.go @@ -181,3 +181,13 @@ func maybePrependRegexPrefix(path, controllerPrefix string, applyLegacyHeuristic } return path } + +func applyExpressionToIngressRules(result *ingressRules) { + for _, svc := range result.ServiceNameToServices { + for i := range svc.Routes { + translators.ApplyExpressionToL4KongRoute(&svc.Routes[i]) + svc.Routes[i].Destinations = nil + svc.Routes[i].SNIs = nil + } + } +} diff --git a/internal/dataplane/parser/translators/l4route_atc.go b/internal/dataplane/parser/translators/l4route_atc.go new file mode 100644 index 0000000000..fd73ca5f9d --- /dev/null +++ b/internal/dataplane/parser/translators/l4route_atc.go @@ -0,0 +1,27 @@ +package translators + +import ( + "github.com/samber/lo" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/kongstate" + "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/parser/atc" +) + +// ApplyExpressionToL4KongRoute convert route flavor from traditional to expressions +// against protocols, snis and dest ports. +func ApplyExpressionToL4KongRoute(r *kongstate.Route) { + matchers := []atc.Matcher{} + + sniMatcher := sniMatcherFromSNIs(lo.Map(r.Route.SNIs, func(item *string, _ int) string { return *item })) + matchers = append(matchers, sniMatcher) + + // TODO(rodman10): replace with helper function. + portMatchers := make([]atc.Matcher, 0, len(r.Destinations)) + for _, dst := range r.Destinations { + portMatchers = append(portMatchers, atc.NewPredicate(atc.FieldNetDstPort, atc.OpEqual, atc.IntLiteral(*dst.Port))) + } + matchers = append(matchers, atc.Or(portMatchers...)) + + r.ExpressionRoutes = true + atc.ApplyExpression(&r.Route, atc.And(matchers...), 1) +} diff --git a/test/integration/tcproute_test.go b/test/integration/tcproute_test.go index e08a2d3624..b0d705bbd4 100644 --- a/test/integration/tcproute_test.go +++ b/test/integration/tcproute_test.go @@ -29,7 +29,6 @@ import ( ) func TestTCPRouteEssentials(t *testing.T) { - skipTestForExpressionRouter(t) ctx := context.Background() t.Log("locking TCP port") @@ -412,7 +411,6 @@ func TestTCPRouteEssentials(t *testing.T) { } func TestTCPRouteReferenceGrant(t *testing.T) { - skipTestForExpressionRouter(t) ctx := context.Background() t.Log("locking TCP port")