From 3928edba8e8dadb0c746eb603dd6b5a13bc76355 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Thu, 28 Mar 2024 11:27:44 -0300 Subject: [PATCH 1/2] add TCPRoute support from Gateway API --- .../en/docs/configuration/gateway-api.md | 10 +- pkg/controller/config/config.go | 16 +- pkg/controller/config/options.go | 2 +- pkg/controller/legacy/cache.go | 5 + pkg/controller/reconciler/watchers.go | 16 ++ pkg/controller/services/cache.go | 17 ++ pkg/controller/services/services.go | 1 + pkg/converters/gateway/gateway.go | 155 ++++++++++++++---- pkg/converters/gateway/gateway_test.go | 101 ++++++++++-- pkg/converters/helper_test/cachemock.go | 5 + pkg/converters/types/interfaces.go | 2 + pkg/converters/types/options.go | 1 + tests/framework/framework.go | 106 +++++++++++- tests/framework/options/objects.go | 31 +++- tests/integration/integration_test.go | 38 +++-- 15 files changed, 437 insertions(+), 69 deletions(-) diff --git a/docs/content/en/docs/configuration/gateway-api.md b/docs/content/en/docs/configuration/gateway-api.md index 9c95ce50..66b717a1 100644 --- a/docs/content/en/docs/configuration/gateway-api.md +++ b/docs/content/en/docs/configuration/gateway-api.md @@ -12,8 +12,8 @@ description: > The following steps configure the Kubernetes cluster and HAProxy Ingress to read and parse Gateway API resources: -* Manually install the Gateway API CRDs from the standard channel. See the Gateway API [documentation](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) - * ... or simply `kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml` +* Manually install the Gateway API CRDs from the experimental channel - HAProxy Ingress supports TCPRoute which is not included in the standard channel. See the Gateway API [documentation](https://gateway-api.sigs.k8s.io/guides/#installing-gateway-api) + * ... or simply `kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml` * `v1.0.0` is just a reference for a fresh new deployment, Gateway API `v0.4.0` or any newer versions are supported. * Start (or restart) the controller @@ -25,9 +25,9 @@ Gateway API `v1alpha2`, `v1beta1` and `v1` specs are partially implemented in v0 * Target Services can be annotated with [Backend or Path scoped]({{% relref "keys#scope" %}}) configuration keys, this will continue to be supported. * Gateway API resources doesn't support annotations, this is planned to continue to be unsupported. Extensions to the Gateway API spec will be added in the extension points of the API. -* Only the `GatewayClass`, `Gateway` and `HTTPRoute` resource definitions are implemented. +* Only the `GatewayClass`, `Gateway`, `TCPRoute` and `HTTPRoute` resource definitions are implemented. * The controller doesn't implement partial parsing yet for Gateway API resources, changes should be a bit slow on clusters with thousands of Ingress, Gateway API resources or Services. -* Gateway's Listener Port and Protocol are not implemented - Port uses the global [bind-port]({{% relref "keys#bind-port" %}}) configuration and Protocol is based on the presence or absence of the TLS attribute. +* Gateway's Listener Port and Protocol are implemented for TCPRoute, but they are not implemented for HTTPRoute - for HTTP workloads, Port uses the global [bind-port]({{% relref "keys#bind-port" %}}) configuration and Protocol is based on the presence or absence of the TLS attribute. * Gateway's Addresses is not implemented - binding addresses use the global [bind-ip-addr]({{% relref "keys#bind-ip-addr" %}}) configuration. * Gateway's Hostname only supports empty/absence of Hostname or a single `*`, any other string will override the HTTPRoute Hostnames configuration without any merging. * HTTPRoute's Rules and BackendRefs don't support Filters. @@ -51,7 +51,7 @@ Add the following steps to the [Getting Started guide]({{% relref "/docs/getting [Manually install](https://gateway-api.sigs.k8s.io/v1alpha2/guides/getting-started/#installing-gateway-api-crds-manually) the Gateway API CRDs: ``` -kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml ``` Add the following deployment and service if echoserver isn't running yet: diff --git a/pkg/controller/config/config.go b/pkg/controller/config/config.go index 11ba169d..1648daf2 100644 --- a/pkg/controller/config/config.go +++ b/pkg/controller/config/config.go @@ -202,9 +202,10 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options configLog.Info("watching for Gateway API resources - --watch-gateway is true") } - var hasGatewayV1, hasGatewayB1, hasGatewayA2 bool + var hasGatewayV1, hasGatewayB1, hasGatewayA2, hasTCPRouteA2 bool if opt.WatchGateway { gwapis := []string{"gatewayclass", "gateway", "httproute"} + tcpapis := []string{"tcproute"} gwV1 := configHasAPI(clientGateway.Discovery(), gatewayv1.GroupVersion, gwapis...) if gwV1 { @@ -221,9 +222,20 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options // only one GatewayClass/Gateway/HTTPRoute version should be enabled at the same time, // otherwise we'd be retrieving the same duplicated resource from distinct api endpoints. + gw := gwV1 || gwB1 || gwA2 hasGatewayV1 = gwV1 hasGatewayB1 = gwB1 && !hasGatewayV1 hasGatewayA2 = gwA2 && !hasGatewayB1 + + tcpA2 := configHasAPI(clientGateway.Discovery(), gatewayv1alpha2.GroupVersion, tcpapis...) + if tcpA2 { + configLog.Info("found custom resource definition for TCPRoute API v1alpha2") + } + + // TODO: cannot enable TCPRoute without Gateway and GatewayClass, but currently HTTPRoute + // discovery is coupled and its CRD should be installed as well, even if not used. + // We should use a distinct flag for HTTPRoute. + hasTCPRouteA2 = tcpA2 && gw } if opt.EnableEndpointSlicesAPI { @@ -475,6 +487,7 @@ func CreateWithConfig(ctx context.Context, restConfig *rest.Config, opt *Options HasGatewayA2: hasGatewayA2, HasGatewayB1: hasGatewayB1, HasGatewayV1: hasGatewayV1, + HasTCPRouteA2: hasTCPRouteA2, HealthzAddr: healthz, HealthzURL: opt.HealthzURL, IngressClass: opt.IngressClass, @@ -657,6 +670,7 @@ type Config struct { HasGatewayA2 bool HasGatewayB1 bool HasGatewayV1 bool + HasTCPRouteA2 bool HealthzAddr string HealthzURL string IngressClass string diff --git a/pkg/controller/config/options.go b/pkg/controller/config/options.go index 77827543..d706b257 100644 --- a/pkg/controller/config/options.go +++ b/pkg/controller/config/options.go @@ -201,7 +201,7 @@ func (o *Options) AddFlags(fs *flag.FlagSet) { "processes.", ) - flag.StringVar(&o.MasterSocket, "master-socket", o.MasterSocket, ""+ + fs.StringVar(&o.MasterSocket, "master-socket", o.MasterSocket, ""+ "Defines the master CLI unix socket of an external HAProxy running in "+ "master-worker mode. Defaults to use the embedded HAProxy if not declared.", ) diff --git a/pkg/controller/legacy/cache.go b/pkg/controller/legacy/cache.go index afeeaabb..97c2a2d8 100644 --- a/pkg/controller/legacy/cache.go +++ b/pkg/controller/legacy/cache.go @@ -222,6 +222,7 @@ func (c *k8scache) hasGateway() bool { var errGatewayA2Disabled = fmt.Errorf("gateway API v1alpha2 wasn't initialized") var errGatewayB1Disabled = fmt.Errorf("legacy controller does not support Gateway API v1beta1") var errGatewayV1Disabled = fmt.Errorf("legacy controller does not support Gateway API v1") +var errTCPRouteA2Disabled = fmt.Errorf("legacy controller does not support TCPRoute API") func (c *k8scache) GetGatewayA2(namespace, name string) (*gatewayv1alpha2.Gateway, error) { if !c.hasGateway() { @@ -272,6 +273,10 @@ func (c *k8scache) GetHTTPRouteList() ([]*gatewayv1.HTTPRoute, error) { return nil, errGatewayV1Disabled } +func (c *k8scache) GetTCPRouteList() ([]*gatewayv1alpha2.TCPRoute, error) { + return nil, errTCPRouteA2Disabled +} + func (c *k8scache) GetService(defaultNamespace, serviceName string) (*api.Service, error) { namespace, name, err := c.buildResourceName(defaultNamespace, "service", serviceName, c.dynamicConfig.CrossNamespaceServices) if err != nil { diff --git a/pkg/controller/reconciler/watchers.go b/pkg/controller/reconciler/watchers.go index 5b2a0091..65394837 100644 --- a/pkg/controller/reconciler/watchers.go +++ b/pkg/controller/reconciler/watchers.go @@ -74,6 +74,9 @@ func (w *watchers) getHandlers() []*hdlr { if w.cfg.HasGatewayV1 { handlers = append(handlers, w.handlersGatewayv1()...) } + if w.cfg.HasTCPRouteA2 { + handlers = append(handlers, w.handlersTCPRoutev1alpha2()...) + } for _, h := range handlers { h.w = w } @@ -476,6 +479,19 @@ func (w *watchers) handlersGatewayv1() []*hdlr { } } +func (w *watchers) handlersTCPRoutev1alpha2() []*hdlr { + return []*hdlr{ + { + typ: &gatewayv1alpha2.TCPRoute{}, + res: types.ResourceTCPRoute, + full: true, + pr: []predicate.Predicate{ + predicate.GenerationChangedPredicate{}, + }, + }, + } +} + type hdlr struct { w *watchers typ client.Object diff --git a/pkg/controller/services/cache.go b/pkg/controller/services/cache.go index 5d75986a..c4879d86 100644 --- a/pkg/controller/services/cache.go +++ b/pkg/controller/services/cache.go @@ -72,6 +72,7 @@ type c struct { var errGatewayA2Disabled = fmt.Errorf("gateway API v1alpha2 wasn't initialized") var errGatewayB1Disabled = fmt.Errorf("gateway API v1beta1 wasn't initialized") var errGatewayV1Disabled = fmt.Errorf("gateway API v1 wasn't initialized") +var errTCPRouteA2Disabled = fmt.Errorf("TCPRoute API v1alpha2 wasn't initialized") func (c *c) get(key string, obj client.Object) error { ns, n, err := cache.SplitMetaNamespaceKey(key) @@ -381,6 +382,22 @@ func (c *c) GetHTTPRouteList() ([]*gatewayv1.HTTPRoute, error) { return rlist, nil } +func (c *c) GetTCPRouteList() ([]*gatewayv1alpha2.TCPRoute, error) { + if !c.config.HasTCPRouteA2 { + return nil, errTCPRouteA2Disabled + } + list := gatewayv1alpha2.TCPRouteList{} + err := c.client.List(c.ctx, &list) + if err != nil { + return nil, err + } + rlist := make([]*gatewayv1alpha2.TCPRoute, len(list.Items)) + for i := range list.Items { + rlist[i] = &list.Items[i] + } + return rlist, nil +} + func (c *c) GetService(defaultNamespace, serviceName string) (*api.Service, error) { namespace, name, err := buildResourceName(defaultNamespace, "service", serviceName, c.dynconfig.CrossNamespaceServices) if err != nil { diff --git a/pkg/controller/services/services.go b/pkg/controller/services/services.go index d7b737a1..abf33aeb 100644 --- a/pkg/controller/services/services.go +++ b/pkg/controller/services/services.go @@ -165,6 +165,7 @@ func (s *Services) setup(ctx context.Context) error { HasGatewayA2: cfg.HasGatewayA2, HasGatewayB1: cfg.HasGatewayB1, HasGatewayV1: cfg.HasGatewayV1, + HasTCPRouteA2: cfg.HasTCPRouteA2, EnableEPSlices: cfg.EnableEndpointSliceAPI, } instance := haproxy.CreateInstance(s.legacylogger.new("haproxy"), instanceOptions) diff --git a/pkg/converters/gateway/gateway.go b/pkg/converters/gateway/gateway.go index 3df4b586..6a59cadd 100644 --- a/pkg/converters/gateway/gateway.go +++ b/pkg/converters/gateway/gateway.go @@ -81,6 +81,11 @@ func (c *converter) Sync(full bool, gwtyp client.Object) { return } + c.syncHTTPRoutes(gwtyp) + c.syncTCPRoutes(gwtyp) +} + +func (c *converter) syncHTTPRoutes(gwtyp client.Object) { var httpRoutesSource []*httpRouteSource var err error switch gwtyp.(type) { @@ -100,7 +105,9 @@ func (c *converter) Sync(full bool, gwtyp client.Object) { sortHTTPRoutes(httpRoutesSource) for _, httpRoute := range httpRoutesSource { - c.syncHTTPRoute(httpRoute, gwtyp) + c.syncRoute(&httpRoute.source, httpRoute.spec.ParentRefs, gwtyp, func(gatewaySource *gatewaySource, sectionName *gatewayv1.SectionName) error { + return c.syncHTTPRouteGateway(httpRoute, gatewaySource, sectionName) + }) } } @@ -140,6 +147,27 @@ func (c *converter) getHTTPRoutesSource() ([]*httpRouteSource, error) { return httpRoutesSource, nil } +func (c *converter) syncTCPRoutes(gwtyp client.Object) { + if !c.options.HasTCPRouteA2 { + return + } + tcpRoutes, err := c.cache.GetTCPRouteList() + if err != nil { + c.logger.Warn("error reading tcpRoute list: %v", err) + return + } + tcpRoutesSource := make([]*tcpRouteSource, len(tcpRoutes)) + for i := range tcpRoutes { + tcpRoutesSource[i] = newTCPRouteSource(tcpRoutes[i], &tcpRoutes[i].Spec) + } + sortTCPRoutes(tcpRoutesSource) + for _, tcpRoute := range tcpRoutesSource { + c.syncRoute(&tcpRoute.source, tcpRoute.spec.ParentRefs, gwtyp, func(gatewaySource *gatewaySource, sectionName *gatewayv1.SectionName) error { + return c.syncTCPRouteGateway(tcpRoute, gatewaySource, sectionName) + }) + } +} + func sortHTTPRoutes(httpRoutesSource []*httpRouteSource) { sort.Slice(httpRoutesSource, func(i, j int) bool { h1 := httpRoutesSource[i].obj @@ -151,6 +179,17 @@ func sortHTTPRoutes(httpRoutesSource []*httpRouteSource) { }) } +func sortTCPRoutes(tcpRoutesSource []*tcpRouteSource) { + sort.Slice(tcpRoutesSource, func(i, j int) bool { + r1 := tcpRoutesSource[i].obj + r2 := tcpRoutesSource[j].obj + if r1.GetCreationTimestamp() != r2.GetCreationTimestamp() { + return r1.GetCreationTimestamp().Time.Before(r2.GetCreationTimestamp().Time) + } + return r1.GetNamespace()+"/"+r1.GetName() < r2.GetNamespace()+"/"+r2.GetName() + }) +} + type source struct { obj client.Object // @@ -162,6 +201,11 @@ type httpRouteSource struct { spec *gatewayv1.HTTPRouteSpec } +type tcpRouteSource struct { + source + spec *gatewayv1alpha2.TCPRouteSpec +} + type gatewaySource struct { source spec *gatewayv1.GatewaySpec @@ -187,6 +231,13 @@ func newHTTPRouteSource(obj client.Object, spec *gatewayv1.HTTPRouteSpec) *httpR } } +func newTCPRouteSource(obj client.Object, spec *gatewayv1alpha2.TCPRouteSpec) *tcpRouteSource { + return &tcpRouteSource{ + spec: spec, + source: newSource(obj), + } +} + func (c *converter) newGatewaySource(namespace, name string, gwtyp client.Object) *gatewaySource { // TODO: we can simplify all these abstract gw/route fetching code after v0.16, // when the old controller is going to be dropped and we can redesign the cache interface. @@ -220,8 +271,8 @@ var ( gatewayKind = gatewayv1.Kind("Gateway") ) -func (c *converter) syncHTTPRoute(httpRouteSource *httpRouteSource, gwtyp client.Object) { - for _, parentRef := range httpRouteSource.spec.ParentRefs { +func (c *converter) syncRoute(routeSource *source, parentRefs []gatewayv1.ParentReference, gwtyp client.Object, syncGateway func(gatewaySource *gatewaySource, sectionName *gatewayv1.SectionName) error) { + for _, parentRef := range parentRefs { parentGroup := gatewayGroup parentKind := gatewayKind if parentRef.Group != nil && *parentRef.Group != "" { @@ -232,10 +283,10 @@ func (c *converter) syncHTTPRoute(httpRouteSource *httpRouteSource, gwtyp client } if parentGroup != gatewayGroup || parentKind != gatewayKind { c.logger.Warn("ignoring unsupported Group/Kind reference on %s: %s/%s", - httpRouteSource, parentGroup, parentKind) + routeSource, parentGroup, parentKind) continue } - namespace := httpRouteSource.namespace + namespace := routeSource.namespace if parentRef.Namespace != nil && *parentRef.Namespace != "" { namespace = string(*parentRef.Namespace) } @@ -244,9 +295,9 @@ func (c *converter) syncHTTPRoute(httpRouteSource *httpRouteSource, gwtyp client continue } // TODO implement gateway.Spec.Addresses - err := c.syncHTTPRouteGateway(httpRouteSource, gatewaySource, parentRef.SectionName) + err := syncGateway(gatewaySource, parentRef.SectionName) if err != nil { - c.logger.Warn("cannot attach %s to %s: %s", httpRouteSource, gatewaySource, err) + c.logger.Warn("cannot attach %s to %s: %s", routeSource, gatewaySource, err) } } } @@ -256,21 +307,25 @@ func (c *converter) syncHTTPRouteGateway(httpRouteSource *httpRouteSource, gatew if sectionName != nil && *sectionName != listener.Name { continue } - if err := c.checkListenerAllowed(gatewaySource, httpRouteSource, &listener); err != nil { + if err := c.checkListenerAllowed(gatewaySource, &httpRouteSource.source, &listener); err != nil { c.logger.Warn("skipping attachment of %s to %s listener '%s': %s", httpRouteSource, gatewaySource, listener.Name, err) continue } for index, rule := range httpRouteSource.spec.Rules { // TODO implement rule.Filters - backend, services := c.createBackend(httpRouteSource, fmt.Sprintf("_rule%d", index), rule.BackendRefs) + backendRefs := make([]gatewayv1.BackendRef, len(rule.BackendRefs)) + for i := range rule.BackendRefs { + backendRefs[i] = rule.BackendRefs[i].BackendRef + } + backend, services := c.createBackend(&httpRouteSource.source, fmt.Sprintf("_rule%d", index), backendRefs) if backend != nil { passthrough := listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatewayv1.TLSModePassthrough if passthrough { backend.ModeTCP = true } hostnames := c.filterHostnames(listener.Hostname, httpRouteSource.spec.Hostnames) - hosts, pathLinks := c.createHTTPHosts(httpRouteSource, hostnames, rule.Matches, backend) + hosts, pathLinks := c.createHTTPHosts(&httpRouteSource.source, hostnames, rule.Matches, backend) c.applyCertRef(gatewaySource, &listener, hosts) if c.ann != nil { c.ann.ReadAnnotations(backend, services, pathLinks) @@ -281,9 +336,33 @@ func (c *converter) syncHTTPRouteGateway(httpRouteSource *httpRouteSource, gatew return nil } +func (c *converter) syncTCPRouteGateway(tcpRouteSource *tcpRouteSource, gatewaySource *gatewaySource, sectionName *gatewayv1.SectionName) error { + for _, listener := range gatewaySource.spec.Listeners { + if sectionName != nil && *sectionName != listener.Name { + continue + } + if err := c.checkListenerAllowed(gatewaySource, &tcpRouteSource.source, &listener); err != nil { + c.logger.Warn("skipping attachment of %s to %s listener '%s': %s", + tcpRouteSource, gatewaySource, listener.Name, err) + continue + } + for index, rule := range tcpRouteSource.spec.Rules { + // TODO implement rule.Filters + backend, services := c.createBackend(&tcpRouteSource.source, fmt.Sprintf("_tcprule%d", index), rule.BackendRefs) + if backend != nil { + pathLinks := c.createTCPService(listener.Port, backend) + if c.ann != nil { + c.ann.ReadAnnotations(backend, services, pathLinks) + } + } + } + } + return nil +} + var errRouteNotAllowed = fmt.Errorf("listener does not allow the route") -func (c *converter) checkListenerAllowed(gatewaySource *gatewaySource, routeSource *httpRouteSource, listener *gatewayv1.Listener) error { +func (c *converter) checkListenerAllowed(gatewaySource *gatewaySource, routeSource *source, listener *gatewayv1.Listener) error { if listener == nil || listener.AllowedRoutes == nil { return errRouteNotAllowed } @@ -296,7 +375,7 @@ func (c *converter) checkListenerAllowed(gatewaySource *gatewaySource, routeSour return nil } -func checkListenerAllowedKind(routeSource *httpRouteSource, kinds []gatewayv1.RouteGroupKind) error { +func checkListenerAllowedKind(routeSource *source, kinds []gatewayv1.RouteGroupKind) error { if len(kinds) == 0 { return nil } @@ -308,7 +387,7 @@ func checkListenerAllowedKind(routeSource *httpRouteSource, kinds []gatewayv1.Ro return fmt.Errorf("listener does not allow route of Kind '%s'", routeSource.kind) } -func (c *converter) checkListenerAllowedNamespace(gatewaySource *gatewaySource, routeSource *httpRouteSource, namespaces *gatewayv1.RouteNamespaces) error { +func (c *converter) checkListenerAllowedNamespace(gatewaySource *gatewaySource, routeSource *source, namespaces *gatewayv1.RouteNamespaces) error { if namespaces == nil || namespaces.From == nil { return errRouteNotAllowed } @@ -337,8 +416,8 @@ func (c *converter) checkListenerAllowedNamespace(gatewaySource *gatewaySource, return errRouteNotAllowed } -func (c *converter) createBackend(source *httpRouteSource, index string, backendRefs []gatewayv1.HTTPBackendRef) (*hatypes.Backend, []*api.Service) { - if habackend := c.haproxy.Backends().FindBackend(source.namespace, source.name, index); habackend != nil { +func (c *converter) createBackend(routeSource *source, index string, backendRefs []gatewayv1.BackendRef) (*hatypes.Backend, []*api.Service) { + if habackend := c.haproxy.Backends().FindBackend(routeSource.namespace, routeSource.name, index); habackend != nil { return habackend, nil } type backend struct { @@ -357,26 +436,26 @@ func (c *converter) createBackend(source *httpRouteSource, index string, backend // TODO implement back.Group // TODO implement back.Kind // TODO implement back.Namespace - svcName := source.namespace + "/" + string(back.Name) + svcName := routeSource.namespace + "/" + string(back.Name) c.tracker.TrackRefName([]convtypes.TrackingRef{ {Context: convtypes.ResourceService, UniqueName: svcName}, {Context: convtypes.ResourceEndpoints, UniqueName: svcName}, }, convtypes.ResourceGateway, "gw") svc, err := c.cache.GetService("", svcName) if err != nil { - c.logger.Warn("skipping service '%s' on %s: %v", back.Name, source, err) + c.logger.Warn("skipping service '%s' on %s: %v", back.Name, routeSource, err) continue } svclist = append(svclist, svc) portStr := strconv.Itoa(int(*back.Port)) svcport := convutils.FindServicePort(svc, portStr) if svcport == nil { - c.logger.Warn("skipping service '%s' on %s: port '%s' not found", back.Name, source, portStr) + c.logger.Warn("skipping service '%s' on %s: port '%s' not found", back.Name, routeSource, portStr) continue } epready, _, err := convutils.CreateEndpoints(c.cache, svc, svcport, c.options.EnableEPSlices) if err != nil { - c.logger.Warn("skipping service '%s' on %s: %v", back.Name, source, err) + c.logger.Warn("skipping service '%s' on %s: %v", back.Name, routeSource, err) continue } weight := 1 @@ -393,12 +472,12 @@ func (c *converter) createBackend(source *httpRouteSource, index string, backend }, }) // TODO implement back.BackendRef - // TODO implement back.Filters + // TODO implement back.Filters (HTTPBackendRef only) } if len(backends) == 0 { return nil, nil } - habackend := c.haproxy.Backends().AcquireBackend(source.namespace, source.name, index) + habackend := c.haproxy.Backends().AcquireBackend(routeSource.namespace, routeSource.name, index) cl := make([]*convutils.WeightCluster, len(backends)) for i := range backends { cl[i] = &backends[i].cl @@ -413,9 +492,9 @@ func (c *converter) createBackend(source *httpRouteSource, index string, backend return habackend, svclist } -func (c *converter) createHTTPHosts(source *httpRouteSource, hostnames []gatewayv1.Hostname, matches []gatewayv1.HTTPRouteMatch, backend *hatypes.Backend) (hosts []*hatypes.Host, pathLinks []*hatypes.PathLink) { +func (c *converter) createHTTPHosts(routeSource *source, hostnames []gatewayv1.Hostname, matches []gatewayv1.HTTPRouteMatch, backend *hatypes.Backend) (hosts []*hatypes.Host, pathLinks []*hatypes.PathLink) { if backend.ModeTCP && len(matches) > 0 { - c.logger.Warn("ignoring match from %s: backend is TCP or SSL Passthrough", source) + c.logger.Warn("ignoring match from %s: backend is TCP or SSL Passthrough", routeSource) matches = nil } if len(matches) == 0 { @@ -463,11 +542,11 @@ func (c *converter) createHTTPHosts(source *httpRouteSource, hostnames []gateway pathlink.WithHeadersMatch(haheaders) if h.FindPathWithLink(pathlink) != nil { if backend.ModeTCP && h.SSLPassthrough() { - c.logger.Warn("skipping redeclared ssl-passthrough root path on %s", source) + c.logger.Warn("skipping redeclared ssl-passthrough root path on %s", routeSource) continue } if !backend.ModeTCP && !h.SSLPassthrough() { - c.logger.Warn("skipping redeclared path '%s' type '%s' on %s", path, haMatch, source) + c.logger.Warn("skipping redeclared path '%s' type '%s' on %s", path, haMatch, routeSource) continue } } @@ -476,19 +555,35 @@ func (c *converter) createHTTPHosts(source *httpRouteSource, hostnames []gateway }, convtypes.ResourceGateway, "gw") h.TLS.UseDefaultCrt = false h.AddLink(backend, pathlink) - c.handlePassthrough(path, h, backend, source) + c.handlePassthrough(path, h, backend, routeSource) hosts = append(hosts, h) pathLinks = append(pathLinks, pathlink) } - // TODO implement match.Headers // TODO implement match.ExtensionRef } return hosts, pathLinks } -func (c *converter) handlePassthrough(path string, h *hatypes.Host, b *hatypes.Backend, source *httpRouteSource) { +func (c *converter) createTCPService(port gatewayv1.PortNumber, backend *hatypes.Backend) []*hatypes.PathLink { + // TODO: this mimics the format currently expected by TCPService, + // implemented by ingress as well; need a refactor, there's already + // a few TODOs in ingress converter and TCPService implementations. + hostname := fmt.Sprintf("%s:%d", hatypes.DefaultHost, port) + backend.ModeTCP = true + _, tcphost := c.haproxy.TCPServices().AcquireTCPService(hostname) + if !tcphost.Backend.IsEmpty() { + c.logger.Warn("skipping redeclared TCPService '%s'", hostname) + return nil + } + c.tracker.TrackNames(convtypes.ResourceHAHostname, hostname, convtypes.ResourceGateway, "gw") + tcphost.Backend = backend.BackendID() + pathLink := hatypes.CreateHostPathLink(hostname, "/", hatypes.MatchExact) + return []*hatypes.PathLink{pathLink} +} + +func (c *converter) handlePassthrough(path string, h *hatypes.Host, b *hatypes.Backend, routeSource *source) { // Special handling for TLS passthrough due to current haproxy.Host limitation - // v0.15 will refactor haproxy.Host, allowing to remove this whole func + // TODO: haproxy.Host refactor, allowing to remove this whole func if path != "/" || (!b.ModeTCP && !h.SSLPassthrough()) { // only matter if root path // we also don't care if both present (b.ModeTCP) and past @@ -510,7 +605,7 @@ func (c *converter) handlePassthrough(path string, h *hatypes.Host, b *hatypes.B h.HTTPPassthroughBackend = b.ID } } else { - c.logger.Warn("skipping redeclared http root path on %s", source) + c.logger.Warn("skipping redeclared http root path on %s", routeSource) } // and // 2. remove it from the target HTTPS configuration diff --git a/pkg/converters/gateway/gateway_test.go b/pkg/converters/gateway/gateway_test.go index 58120b04..a2702945 100644 --- a/pkg/converters/gateway/gateway_test.go +++ b/pkg/converters/gateway/gateway_test.go @@ -18,6 +18,7 @@ package gateway_test import ( "fmt" + "strconv" "strings" "testing" @@ -26,6 +27,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapischeme "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/scheme" "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/gateway" @@ -44,6 +46,7 @@ type testCaseSync struct { expFullSync bool expDefaultHost string expHosts string + expTCPServices string expBackends string expLogging string } @@ -113,7 +116,7 @@ paths: id: "cross-namespace-3", config: func(c *testConfig) { c.createNamespace("ns2", "name=ns2") - c.createGateway1("ns1/web", "l1:name=ns1") + c.createGateway1("ns1/web", "l1::name=ns1") c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") c.createService1("ns2/echoserver", "8080", "172.17.0.11") }, @@ -125,7 +128,7 @@ WARN skipping attachment of HTTPRoute 'ns2/web' to Gateway 'ns1/web' listener 'l id: "cross-namespace-4", config: func(c *testConfig) { c.createNamespace("ns2", "name=ns2") - c.createGateway1("ns1/web", "l1:name=ns2") + c.createGateway1("ns1/web", "l1::name=ns2") c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") c.createService1("ns2/echoserver", "8080", "172.17.0.11") }, @@ -526,6 +529,34 @@ paths: }) } +func TestSyncTCPRouteCore(t *testing.T) { + runTestSync(t, []testCaseSync{ + { + id: "minimum", + config: func(c *testConfig) { + c.createGateway1("default/pg", "l1:5432") + c.createTCPRoute1("default/pg", "pg", "postgres:15432") + c.createService1("default/postgres", "15432", "172.17.0.11") + }, + expTCPServices: ` +- backends: [] + defaultbackend: default_pg__tcprule0 + port: 5432 + proxyprot: false + tls: {} +`, + expBackends: ` +- id: default_pg__tcprule0 + endpoints: + - ip: 172.17.0.11 + port: 15432 + weight: 128 + modetcp: true +`, + }, + }) +} + func TestSyncGatewayTLS(t *testing.T) { defaultBackend := ` - id: default_web__rule0 @@ -934,11 +965,15 @@ func runTestSync(t *testing.T, testCases []testCaseSync) { if test.expHosts == "" { test.expHosts = "[]" } + if test.expTCPServices == "" { + test.expTCPServices = "[]" + } if test.expBackends == "" { test.expBackends = "[]" } c.compareConfigDefaultHost(test.id, test.expDefaultHost) c.compareConfigHosts(test.id, test.expHosts) + c.compareConfigTCPServices(test.id, test.expTCPServices) c.compareConfigBacks(test.id, test.expBackends) } @@ -979,9 +1014,10 @@ func (c *testConfig) sync() { func (c *testConfig) createConverter() gateway.Config { return gateway.NewGatewayConverter( &convtypes.ConverterOptions{ - Cache: c.cache, - Logger: c.logger, - Tracker: c.tracker, + Cache: c.cache, + Logger: c.logger, + Tracker: c.tracker, + HasTCPRouteA2: true, }, c.hconfig, c.cache.SwapChangedObjects(), @@ -1028,13 +1064,21 @@ spec: for _, listener := range strings.Split(listeners, ",") { l := gatewayv1.Listener{} var lname, lselector string - if i := strings.Index(listener, ":"); i >= 0 { - lname = listener[:i] - lselector = listener[i+1:] - } else { - lname = listener + var lport gatewayv1.PortNumber + lsplit := strings.Split(listener, ":") + lname = lsplit[0] + if len(lsplit) > 2 { + lselector = lsplit[2] + } + if len(lsplit) > 1 && lsplit[1] != "" { + port, err := strconv.Atoi(lsplit[1]) + if err != nil { + panic(err) + } + lport = gatewayv1.PortNumber(port) } l.Name = gatewayv1.SectionName(lname) + l.Port = lport from := gatewayv1.NamespacesFromSame var selector *v1.LabelSelector if lselector != "" { @@ -1073,9 +1117,8 @@ func (c *testConfig) createGateway2(name, listeners, secretName string) *gateway return gw } -func (c *testConfig) createHTTPRoute1(name, parent, service string) *gatewayv1.HTTPRoute { - n := strings.Split(name, "/") - var pns, pn, ps string +func splitRouteInfo(name, parent, service string) (n, svc []string, pns, pn, ps string) { + n = strings.Split(name, "/") if i := strings.Index(parent, "/"); i >= 0 { pns = parent[:i] pn = parent[i+1:] @@ -1086,7 +1129,12 @@ func (c *testConfig) createHTTPRoute1(name, parent, service string) *gatewayv1.H ps = pn[i+1:] pn = pn[:i] } - svc := strings.Split(service, ":") + svc = strings.Split(service, ":") + return +} + +func (c *testConfig) createHTTPRoute1(name, parent, service string) *gatewayv1.HTTPRoute { + n, svc, pns, pn, ps := splitRouteInfo(name, parent, service) r := CreateObject(` apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute @@ -1122,6 +1170,27 @@ func (c *testConfig) createHTTPRoute2(name, parent, service, paths string) *gate return r } +func (c *testConfig) createTCPRoute1(name, parent, service string) *gatewayv1alpha2.TCPRoute { + n, svc, pns, pn, ps := splitRouteInfo(name, parent, service) + r := CreateObject(` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: ` + n[1] + ` + namespace: ` + n[0] + ` +spec: + parentRefs: + - name: ` + pn + ` + namespace: ` + pns + ` + sectionName: ` + ps + ` + rules: + - backendRefs: + - name: ` + svc[0] + ` + port: ` + svc[1]).(*gatewayv1alpha2.TCPRoute) + c.cache.TCPRouteList = append(c.cache.TCPRouteList, r) + return r +} + func (c *testConfig) createGatewayResources(res []string) { for _, cfg := range res { obj := CreateObject(cfg) @@ -1168,6 +1237,10 @@ func (c *testConfig) compareConfigHosts(id string, expected string) { c.compareText(id, conv_helper.MarshalHosts(c.hconfig.Hosts().BuildSortedItems()...), expected) } +func (c *testConfig) compareConfigTCPServices(id string, expected string) { + c.compareText(id, conv_helper.MarshalTCPServices(c.hconfig.TCPServices().BuildSortedItems()...), expected) +} + func (c *testConfig) compareConfigBacks(id string, expected string) { c.compareText(id, conv_helper.MarshalBackendsWeight(c.hconfig.Backends().BuildSortedItems()...), expected) } diff --git a/pkg/converters/helper_test/cachemock.go b/pkg/converters/helper_test/cachemock.go index a1ba97ae..6db7c5fd 100644 --- a/pkg/converters/helper_test/cachemock.go +++ b/pkg/converters/helper_test/cachemock.go @@ -47,6 +47,7 @@ type CacheMock struct { SvcList []*api.Service // HTTPRouteList []*gatewayv1.HTTPRoute + TCPRouteList []*gatewayv1alpha2.TCPRoute GatewayList []*gatewayv1.Gateway GatewayClassList []*gatewayv1.GatewayClass // @@ -136,6 +137,10 @@ func (c *CacheMock) GetHTTPRouteList() ([]*gatewayv1.HTTPRoute, error) { return c.HTTPRouteList, nil } +func (c *CacheMock) GetTCPRouteList() ([]*gatewayv1alpha2.TCPRoute, error) { + return c.TCPRouteList, nil +} + // GetGatewayA2 ... func (c *CacheMock) GetGatewayA2(namespace, name string) (*gatewayv1alpha2.Gateway, error) { return nil, fmt.Errorf("missing implementation") diff --git a/pkg/converters/types/interfaces.go b/pkg/converters/types/interfaces.go index ac4983a2..d787f31c 100644 --- a/pkg/converters/types/interfaces.go +++ b/pkg/converters/types/interfaces.go @@ -43,6 +43,7 @@ type Cache interface { GetHTTPRouteA2List() ([]*gatewayv1alpha2.HTTPRoute, error) GetHTTPRouteB1List() ([]*gatewayv1beta1.HTTPRoute, error) GetHTTPRouteList() ([]*gatewayv1.HTTPRoute, error) + GetTCPRouteList() ([]*gatewayv1alpha2.TCPRoute, error) GetService(defaultNamespace, serviceName string) (*api.Service, error) GetEndpoints(service *api.Service) (*api.Endpoints, error) GetConfigMap(configMapName string) (*api.ConfigMap, error) @@ -117,6 +118,7 @@ const ( ResourceGateway ResourceType = "Gateway" ResourceGatewayClass ResourceType = "GatewayClass" ResourceHTTPRoute ResourceType = "HTTPRoute" + ResourceTCPRoute ResourceType = "TCPRoute" ResourceConfigMap ResourceType = "ConfigMap" ResourceService ResourceType = "Service" diff --git a/pkg/converters/types/options.go b/pkg/converters/types/options.go index 00ddda59..3d888d97 100644 --- a/pkg/converters/types/options.go +++ b/pkg/converters/types/options.go @@ -43,6 +43,7 @@ type ConverterOptions struct { HasGatewayA2 bool HasGatewayB1 bool HasGatewayV1 bool + HasTCPRouteA2 bool EnableEPSlices bool } diff --git a/tests/framework/framework.go b/tests/framework/framework.go index 6645e727..2d9688e2 100644 --- a/tests/framework/framework.go +++ b/tests/framework/framework.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "math/rand" + "net" "net/http" "os" "os/exec" @@ -263,6 +264,22 @@ func (f *framework) Request(ctx context.Context, t *testing.T, method, host, pat } } +func (f *framework) TCPRequest(ctx context.Context, t *testing.T, tcpPort int32, data string) string { + var conn net.Conn + require.EventuallyWithT(t, func(collect *assert.CollectT) { + var err error + conn, err = net.Dial("tcp", fmt.Sprintf(":%d", tcpPort)) + assert.NoError(collect, err) + }, 5*time.Second, time.Second) + _, err := conn.Write([]byte(data)) + require.NoError(t, err) + buf := make([]byte, 256) + n, err := conn.Read(buf) + require.NoError(t, err) + conn.Close() + return string(buf[:n]) +} + func (f *framework) Client() client.WithWatch { return f.cli } @@ -448,6 +465,14 @@ func (f *framework) CreateGatewayV1(ctx context.Context, t *testing.T, gc *gatew func (f *framework) CreateGateway(ctx context.Context, t *testing.T, version string, gc *gatewayv1.GatewayClass, o ...options.Object) client.Object { opt := options.ParseObjectOptions(o...) + if opt.Listeners == nil { + opt.Listeners = []options.ListenerOpt{{ + Name: "echoserver-gw", + Port: 80, + Proto: "HTTP", + }} + } + api := v1.GroupVersion{Group: gatewayv1.GroupName, Version: version}.String() data := fmt.Sprintf(` apiVersion: %s @@ -457,10 +482,6 @@ metadata: namespace: default spec: gatewayClassName: "" - listeners: - - protocol: HTTP - port: 80 - name: echoserver-gw `, api) name := randomName("gw") @@ -468,6 +489,13 @@ spec: gw.SetName(name) spec := reflect.ValueOf(gw).Elem().FieldByName("Spec").Addr().Interface().(*gatewayv1.GatewaySpec) spec.GatewayClassName = gatewayv1.ObjectName(gc.Name) + for _, l := range opt.Listeners { + spec.Listeners = append(spec.Listeners, gatewayv1.Listener{ + Name: gatewayv1.SectionName(l.Name), + Protocol: gatewayv1.ProtocolType(l.Proto), + Port: gatewayv1.PortNumber(l.Port), + }) + } opt.Apply(gw) t.Logf("creating Gateway %s/%s\n", gw.GetNamespace(), gw.GetName()) @@ -550,6 +578,57 @@ spec: return route, hostname } +func (f *framework) CreateTCPRouteA2(ctx context.Context, t *testing.T, gw *gatewayv1.Gateway, svc *corev1.Service, o ...options.Object) *gatewayv1alpha2.TCPRoute { + route := f.CreateTCPRoute(ctx, t, gatewayv1alpha2.GroupVersion.Version, gw, svc, o...) + return route.(*gatewayv1alpha2.TCPRoute) +} + +func (f *framework) CreateTCPRoute(ctx context.Context, t *testing.T, version string, gw *gatewayv1.Gateway, svc *corev1.Service, o ...options.Object) client.Object { + opt := options.ParseObjectOptions(o...) + api := v1.GroupVersion{Group: gatewayv1.GroupName, Version: version}.String() + data := fmt.Sprintf(` +apiVersion: %s +kind: TCPRoute +metadata: + name: "" + namespace: default +spec: + parentRefs: + - name: "" + hostnames: + - "" + rules: + - backendRefs: + - name: "" + port: 0 +`, api) + name := randomName("tcproute") + + route := f.CreateObject(t, data) + route.SetName(name) + spec := reflect.ValueOf(route).Elem().FieldByName("Spec").Addr().Interface().(*gatewayv1alpha2.TCPRouteSpec) + spec.ParentRefs[0].Name = gatewayv1.ObjectName(gw.Name) + spec.Rules[0].BackendRefs[0].Name = gatewayv1.ObjectName(svc.Name) + spec.Rules[0].BackendRefs[0].Port = (*gatewayv1.PortNumber)(&svc.Spec.Ports[0].Port) + opt.Apply(route) + + t.Logf("creating TCPRoute %s/%s\n", route.GetNamespace(), route.GetName()) + + err := f.cli.Create(ctx, route) + require.NoError(t, err) + + t.Cleanup(func() { + route := unstructured.Unstructured{} + route.SetAPIVersion(api) + route.SetKind("TCPRoute") + route.SetNamespace("default") + route.SetName(name) + err := f.cli.Delete(ctx, &route) + assert.NoError(t, client.IgnoreNotFound(err)) + }) + return route +} + func (f *framework) CreateObject(t *testing.T, data string) client.Object { obj, _, err := f.codec.UniversalDeserializer().Decode([]byte(data), nil, nil) require.NoError(t, err) @@ -590,6 +669,25 @@ func (f *framework) CreateHTTPServer(ctx context.Context, t *testing.T) int32 { return serverPort } +func (f *framework) CreateTCPServer(ctx context.Context, t *testing.T) int32 { + serverPort := int32(32768 + rand.Intn(32767)) + listen, err := net.Listen("tcp", fmt.Sprintf(":%d", serverPort)) + require.NoError(t, err) + go func() { + for { + conn, err := listen.Accept() + require.NoError(t, err) + buf := make([]byte, 256) + n, err := conn.Read(buf) + require.NoError(t, err) + _, err = conn.Write(buf[:n]) + require.NoError(t, err) + conn.Close() + } + }() + return serverPort +} + func randomName(prefix string) string { return fmt.Sprintf("%s-%08d", prefix, rand.Intn(1e8)) } diff --git a/tests/framework/options/objects.go b/tests/framework/options/objects.go index 256423f1..6da01280 100644 --- a/tests/framework/options/objects.go +++ b/tests/framework/options/objects.go @@ -1,6 +1,10 @@ package options -import "sigs.k8s.io/controller-runtime/pkg/client" +import ( + "math/rand" + + "sigs.k8s.io/controller-runtime/pkg/client" +) type Object func(o *objectOpt) @@ -22,15 +26,40 @@ func DefaultHostTLS() Object { } } +func Listener(name, proto string, port int32) Object { + return func(o *objectOpt) { + o.GatewayOpt.Listeners = append(o.GatewayOpt.Listeners, ListenerOpt{ + Name: name, + Proto: proto, + Port: port, + }) + } +} + +func TCPListener() Object { + return Listener("tcpservice-gw", "TCP", int32(32768+rand.Intn(32767))) +} + type objectOpt struct { Ann map[string]string IngressOpt + GatewayOpt } type IngressOpt struct { DefaultTLS bool } +type GatewayOpt struct { + Listeners []ListenerOpt +} + +type ListenerOpt struct { + Name string + Proto string + Port int32 +} + func (o *objectOpt) Apply(obj client.Object) { ann := obj.GetAnnotations() if ann == nil { diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 873ee747..6ba85c8d 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -23,12 +23,12 @@ func TestIntegrationIngress(t *testing.T) { ctx := context.Background() f := framework.NewFramework(ctx, t) - httpPort := f.CreateHTTPServer(ctx, t) + httpServerPort := f.CreateHTTPServer(ctx, t) lbingpre1 := "127.0.0.1" require.NotEqual(t, framework.PublishAddress, lbingpre1) - svcpre1 := f.CreateService(ctx, t, httpPort) + svcpre1 := f.CreateService(ctx, t, httpServerPort) ingpre1, _ := f.CreateIngress(ctx, t, svcpre1) ingpre1.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{{IP: lbingpre1}} err := f.Client().Status().Update(ctx, ingpre1) @@ -38,7 +38,7 @@ func TestIntegrationIngress(t *testing.T) { t.Run("hello world", func(t *testing.T) { t.Parallel() - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateIngress(ctx, t, svc) res := f.Request(ctx, t, http.MethodGet, hostname, "/", options.ExpectResponseCode(http.StatusOK)) assert.True(t, res.EchoResponse) @@ -47,7 +47,7 @@ func TestIntegrationIngress(t *testing.T) { t.Run("should not redirect to https", func(t *testing.T) { t.Parallel() - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateIngress(ctx, t, svc, options.DefaultHostTLS(), options.AddConfigKeyAnnotations(map[string]string{ingtypes.BackSSLRedirect: "false"}), @@ -58,7 +58,7 @@ func TestIntegrationIngress(t *testing.T) { t.Run("should redirect to https", func(t *testing.T) { t.Parallel() - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateIngress(ctx, t, svc, options.DefaultHostTLS(), options.AddConfigKeyAnnotations(map[string]string{ingtypes.BackSSLRedirect: "true"}), @@ -98,7 +98,7 @@ func TestIntegrationIngress(t *testing.T) { t.Run("should update ingress status", func(t *testing.T) { t.Parallel() - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) ing1, _ := f.CreateIngress(ctx, t, svc) assert.EventuallyWithT(t, func(collect *assert.CollectT) { @@ -122,7 +122,7 @@ func TestIntegrationIngress(t *testing.T) { t.Run("should sync ingress status from publish service", func(t *testing.T) { t.Parallel() - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) ing, _ := f.CreateIngress(ctx, t, svc) // check initial status @@ -201,13 +201,13 @@ func TestIntegrationGateway(t *testing.T) { t.Run("v1alpha2", func(t *testing.T) { f := framework.NewFramework(ctx, t, options.CRDs("gateway-api-v040-v1alpha2")) f.StartController(ctx, t) - httpPort := f.CreateHTTPServer(ctx, t) + httpServerPort := f.CreateHTTPServer(ctx, t) gc := f.CreateGatewayClassA2(ctx, t) t.Run("hello world", func(t *testing.T) { t.Parallel() gw := f.CreateGatewayA2(ctx, t, gc) - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateHTTPRouteA2(ctx, t, gw, svc) res := f.Request(ctx, t, http.MethodGet, hostname, "/", options.ExpectResponseCode(http.StatusOK)) assert.True(t, res.EchoResponse) @@ -218,13 +218,13 @@ func TestIntegrationGateway(t *testing.T) { t.Run("v1beta1", func(t *testing.T) { f := framework.NewFramework(ctx, t, options.CRDs("gateway-api-v050-v1beta1-experimental")) f.StartController(ctx, t) - httpPort := f.CreateHTTPServer(ctx, t) + httpServerPort := f.CreateHTTPServer(ctx, t) gc := f.CreateGatewayClassB1(ctx, t) t.Run("hello world", func(t *testing.T) { t.Parallel() gw := f.CreateGatewayB1(ctx, t, gc) - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateHTTPRouteB1(ctx, t, gw, svc) res := f.Request(ctx, t, http.MethodGet, hostname, "/", options.ExpectResponseCode(http.StatusOK)) assert.True(t, res.EchoResponse) @@ -235,17 +235,29 @@ func TestIntegrationGateway(t *testing.T) { t.Run("v1", func(t *testing.T) { f := framework.NewFramework(ctx, t, options.CRDs("gateway-api-v100-v1-experimental")) f.StartController(ctx, t) - httpPort := f.CreateHTTPServer(ctx, t) + httpServerPort := f.CreateHTTPServer(ctx, t) + tcpServerPort := f.CreateTCPServer(ctx, t) gc := f.CreateGatewayClassV1(ctx, t) t.Run("hello world", func(t *testing.T) { t.Parallel() gw := f.CreateGatewayV1(ctx, t, gc) - svc := f.CreateService(ctx, t, httpPort) + svc := f.CreateService(ctx, t, httpServerPort) _, hostname := f.CreateHTTPRouteV1(ctx, t, gw, svc) res := f.Request(ctx, t, http.MethodGet, hostname, "/", options.ExpectResponseCode(http.StatusOK)) assert.True(t, res.EchoResponse) assert.Equal(t, "http", res.ReqHeaders["x-forwarded-proto"]) }) + + t.Run("expose TCPRoute", func(t *testing.T) { + t.Parallel() + gw := f.CreateGatewayV1(ctx, t, gc, options.Listener("pgserver", "TCP", 15432)) + svc := f.CreateService(ctx, t, tcpServerPort) + _ = f.CreateTCPRouteA2(ctx, t, gw, svc) + res1 := f.TCPRequest(ctx, t, 15432, "ping") + assert.Equal(t, "ping", res1) + res2 := f.TCPRequest(ctx, t, 15432, "reply") + assert.Equal(t, "reply", res2) + }) }) } From d91f395f5923868f646be5bcf27445fe240c8b3e Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Fri, 29 Mar 2024 14:46:21 -0300 Subject: [PATCH 2/2] doc: TCPRoute example --- .../en/docs/configuration/gateway-api.md | 85 +++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/docs/content/en/docs/configuration/gateway-api.md b/docs/content/en/docs/configuration/gateway-api.md index 66b717a1..bf288b3e 100644 --- a/docs/content/en/docs/configuration/gateway-api.md +++ b/docs/content/en/docs/configuration/gateway-api.md @@ -54,11 +54,10 @@ Add the following steps to the [Getting Started guide]({{% relref "/docs/getting kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml ``` -Add the following deployment and service if echoserver isn't running yet: +Restart HAProxy Ingress so it can find the just installed APIs: ``` -kubectl --namespace default create deployment echoserver --image k8s.gcr.io/echoserver:1.3 -kubectl --namespace default expose deployment echoserver --port=8080 +kubectl --namespace ingress-controller delete pod -lapp.kubernetes.io/name=haproxy-ingress ``` A GatewayClass enables Gateways to be read and parsed by HAProxy Ingress. Create a GatewayClass with the following content: @@ -72,7 +71,16 @@ spec: controllerName: haproxy-ingress.github.io/controller ``` -Gateways create listeners and allow to configure hostnames. Create a Gateway with the following content: +### Deploy HTTP workload + +Add the following deployment and service if echoserver isn't running yet: + +``` +kubectl --namespace default create deployment echoserver --image k8s.gcr.io/echoserver:1.3 +kubectl --namespace default expose deployment echoserver --port=8080 +``` + +Gateways create listeners and allow to configure hostnames for HTTP workloads. Create a Gateway with the following content: Note: port and protocol attributes [have some limitations](#conformance). @@ -96,8 +104,6 @@ HTTPRoutes configure the hostnames and target services. Create a HTTPRoute with apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: - labels: - gateway: echo name: echoserver namespace: default spec: @@ -117,3 +123,70 @@ Send a request to our just configured route: curl http://echoserver-from-gateway.local wget -qO- http://echoserver-from-gateway.local ``` + +### Deploy TCP workload + +Add the following deployment and service: + +``` +kubectl --namespace default create deployment redis --image docker.io/redis +kubectl --namespace default expose deployment redis --port=6379 +``` + +A new port need to be added if HAProxy Ingress is not configured in the host network. If so, add the following snippet in `values.yaml` and apply it using Helm: + +``` +controller: + ... + service: + ... + extraPorts: + - port: 6379 + targetPort: 6379 +``` + +Gateways create listeners and allow to configure the listening port for TCP workloads. Create a Gateway with the following content: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: redis + namespace: default +spec: + gatewayClassName: haproxy + listeners: + - name: redis-gw + port: 6379 + protocol: TCP +``` + +TCPRoutes configure the target services. Create a TCPRoute with the following content: + +```yaml +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: redis + namespace: default +spec: + parentRefs: + - name: redis + rules: + - backendRefs: + - name: redis + port: 6379 +``` + +Send a ping to the Redis server using `curl`. Change `192.168.106.2` below to the IP address of HAProxy Ingress: + +``` +curl -v telnet://192.168.106.2:6379 +* Trying 192.168.106.2:6379... +* Connected to 192.168.106.2 (192.168.106.2) port 6379 +ping ++PONG +^C +``` + +Type `ping` and see a `+PONG` response. Press `^C` to close the connection.