From 50662eee7402f0f03030945a3994140194ed984f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 21:39:19 +0000 Subject: [PATCH 1/4] Initial plan From 5dc8a9328cb023ec40c6ff7ca9f9d5670795c8ef Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 21:45:15 +0000 Subject: [PATCH 2/4] Fix 404 errors: add backend-protocol support for HTTPS backends and improve route defaults Co-authored-by: bwalsh <47808+bwalsh@users.noreply.github.com> --- .../overlays/ingress-authz-overlay/README.md | 34 +++++++++++++++++++ .../docs/authz-ingress-user-guide.md | 34 ++++++++++++++++++- .../templates/externalname-services.yaml | 3 +- .../templates/ingress-authz.yaml | 16 ++++++++- .../ingress-authz-overlay/values.yaml | 2 ++ helm/argo-stack/values.yaml | 2 ++ 6 files changed, 88 insertions(+), 3 deletions(-) diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/README.md b/helm/argo-stack/overlays/ingress-authz-overlay/README.md index 20a12cb3..aa088a2c 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/README.md +++ b/helm/argo-stack/overlays/ingress-authz-overlay/README.md @@ -117,6 +117,40 @@ ingressAuthzOverlay: clusterIssuer: letsencrypt-prod ``` +### Route Configuration Options + +Each route supports the following options: + +| Option | Description | Default | +|--------|-------------|---------| +| `enabled` | Enable/disable this route | `true` | +| `namespace` | Namespace where the ingress is created | Required | +| `service` | Backend service name | Required | +| `serviceNamespace` | Namespace of the actual service (for cross-namespace routing) | Same as `namespace` | +| `port` | Backend service port | Required | +| `pathPrefix` | URL path prefix for this route | Required | +| `useRegex` | Enable regex path matching | `false` | +| `rewriteTarget` | Path rewrite target (when `useRegex` is true) | `/$2` | +| `backendProtocol` | Backend protocol (`HTTP`, `HTTPS`, `GRPC`, `GRPCS`) | `HTTP` | +| `proxyConnectTimeout` | NGINX proxy connect timeout | - | +| `proxyReadTimeout` | NGINX proxy read timeout | - | +| `proxySendTimeout` | NGINX proxy send timeout | - | + +### Backend Protocol + +Some services use HTTPS or gRPC internally. Use the `backendProtocol` option to specify the correct protocol: + +```yaml +ingressAuthzOverlay: + routes: + applications: + # ArgoCD server uses HTTPS by default + backendProtocol: HTTPS + grpc-service: + # For gRPC services + backendProtocol: GRPC +``` + ## Documentation - [User Guide](docs/authz-ingress-user-guide.md) - Complete installation and configuration guide diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/docs/authz-ingress-user-guide.md b/helm/argo-stack/overlays/ingress-authz-overlay/docs/authz-ingress-user-guide.md index d97c5989..b553e1c3 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/docs/authz-ingress-user-guide.md +++ b/helm/argo-stack/overlays/ingress-authz-overlay/docs/authz-ingress-user-guide.md @@ -522,7 +522,39 @@ curl -I -H "Authorization: Bearer $TOKEN" https://calypr-demo.ddns.net/workflows ### Common Issues -1. **502 Bad Gateway**: AuthZ adapter not reachable +1. **404 Not Found**: Routes return 404 error + - Check if backend services exist in their respective namespaces: + ```bash + # For workflows + kubectl get svc argo-stack-argo-workflows-server -n argo-workflows + # For applications (ArgoCD) + kubectl get svc argo-stack-argocd-server -n argocd + # For registrations + kubectl get svc github-repo-registrations-eventsource-svc -n argo-events + ``` + - Verify ExternalName proxy services are created for cross-namespace routing: + ```bash + kubectl get svc -n argo-stack -l app.kubernetes.io/component=externalname-proxy + ``` + - Check if NGINX ingress controller is running and has ADDRESS assigned: + ```bash + kubectl get ingress -A -l app.kubernetes.io/name=ingress-authz-overlay + ``` + - Verify backend protocol settings (ArgoCD requires HTTPS): + ```bash + kubectl get ingress ingress-authz-applications -n argo-stack -o yaml | grep backend-protocol + ``` + - Check NGINX ingress controller logs for routing errors: + ```bash + kubectl logs -n ingress-nginx -l app.kubernetes.io/component=controller --tail=100 + ``` + - Test direct connectivity to backend services: + ```bash + kubectl run -it --rm debug --image=curlimages/curl --restart=Never -- \ + curl -v http://argo-stack-argo-workflows-server.argo-workflows:2746/ + ``` + +2. **502 Bad Gateway**: AuthZ adapter not reachable - Check authz-adapter deployment is running - Verify service selector matches pod labels diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/templates/externalname-services.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/templates/externalname-services.yaml index 7796a3f5..b70fc050 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/templates/externalname-services.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/templates/externalname-services.yaml @@ -8,7 +8,8 @@ in its original namespace. This enables NGINX Ingress to route traffic correctly {{- $root := . }} {{- $config := .Values.ingressAuthzOverlay }} {{- range $routeName, $route := $config.routes }} -{{- if $route.enabled }} +{{- $routeEnabled := $route.enabled | default true }} +{{- if $routeEnabled }} {{- $serviceNamespace := $route.serviceNamespace | default $route.namespace }} {{- if ne $route.namespace $serviceNamespace }} --- diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml index 355446fb..9ee43b4f 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml @@ -17,7 +17,8 @@ externalname-services.yaml template. {{- $root := . }} {{- $config := .Values.ingressAuthzOverlay }} {{- range $routeName, $route := $config.routes }} -{{- if $route.enabled }} +{{- $routeEnabled := $route.enabled | default true }} +{{- if $routeEnabled }} {{- $serviceNamespace := $route.serviceNamespace | default $route.namespace }} {{- $isCrossNamespace := ne $route.namespace $serviceNamespace }} {{- $serviceName := ternary (printf "%s-proxy" $route.service) $route.service $isCrossNamespace }} @@ -45,6 +46,10 @@ metadata: # # Let's Encrypt / cert-manager integration (only on primary route to avoid ownership conflicts) # cert-manager.io/cluster-issuer: {{ $config.tls.clusterIssuer | quote }} # {{- end }} + {{- if $route.backendProtocol }} + # Backend protocol (e.g., HTTPS for ArgoCD, GRPC for gRPC services) + nginx.ingress.kubernetes.io/backend-protocol: {{ $route.backendProtocol | quote }} + {{- end }} {{- if $route.useRegex }} # Path rewriting for subpath support nginx.ingress.kubernetes.io/use-regex: "true" @@ -54,6 +59,15 @@ metadata: # Cross-namespace routing via ExternalName service nginx.ingress.kubernetes.io/upstream-vhost: {{ $route.service }}.{{ $serviceNamespace }}.svc.cluster.local {{- end }} + {{- if $route.proxyConnectTimeout }} + nginx.ingress.kubernetes.io/proxy-connect-timeout: {{ $route.proxyConnectTimeout | quote }} + {{- end }} + {{- if $route.proxyReadTimeout }} + nginx.ingress.kubernetes.io/proxy-read-timeout: {{ $route.proxyReadTimeout | quote }} + {{- end }} + {{- if $route.proxySendTimeout }} + nginx.ingress.kubernetes.io/proxy-send-timeout: {{ $route.proxySendTimeout | quote }} + {{- end }} spec: ingressClassName: {{ $config.ingressClassName | default "nginx" | quote }} {{- if $config.tls.enabled }} diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml index 549dc11b..25117938 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml @@ -126,6 +126,8 @@ ingressAuthzOverlay: pathPrefix: /applications useRegex: true rewriteTarget: /$2 + # ArgoCD server uses HTTPS by default + backendProtocol: HTTPS # GitHub Repository Registrations EventSource registrations: diff --git a/helm/argo-stack/values.yaml b/helm/argo-stack/values.yaml index 85b97848..7df7f077 100644 --- a/helm/argo-stack/values.yaml +++ b/helm/argo-stack/values.yaml @@ -241,6 +241,8 @@ ingressAuthzOverlay: service: argo-stack-argocd-server port: 8080 pathPrefix: /applications + # ArgoCD server uses HTTPS by default + backendProtocol: HTTPS registrations: namespace: argo-stack service: github-repo-registrations-eventsource-svc From 661f586d6ac5916beaee30b40120754b90aaa470 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 21:46:33 +0000 Subject: [PATCH 3/4] Add valid values comment for backendProtocol annotation Co-authored-by: bwalsh <47808+bwalsh@users.noreply.github.com> --- .../ingress-authz-overlay/templates/ingress-authz.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml index 9ee43b4f..f14933e6 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml @@ -47,7 +47,8 @@ metadata: # cert-manager.io/cluster-issuer: {{ $config.tls.clusterIssuer | quote }} # {{- end }} {{- if $route.backendProtocol }} - # Backend protocol (e.g., HTTPS for ArgoCD, GRPC for gRPC services) + # Backend protocol for services using HTTPS/GRPC internally + # Valid values: HTTP, HTTPS, GRPC, GRPCS, AJP, FCGI nginx.ingress.kubernetes.io/backend-protocol: {{ $route.backendProtocol | quote }} {{- end }} {{- if $route.useRegex }} From ba955f0a564544dfdc26b211c312f9a6cd16699a Mon Sep 17 00:00:00 2001 From: Brian Walsh Date: Thu, 27 Nov 2025 04:15:56 +0000 Subject: [PATCH 4/4] fix ingress --- Makefile | 24 ++++++++++++------- .../templates/ingress-authz.yaml | 2 +- .../values-ingress-nginx.yaml | 4 ++++ .../ingress-authz-overlay/values.yaml | 7 +++--- 4 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 helm/argo-stack/overlays/ingress-authz-overlay/values-ingress-nginx.yaml diff --git a/Makefile b/Makefile index d59e7c52..67dcd2c7 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,13 @@ ports: sudo cp /etc/letsencrypt/live/calypr-demo.ddns.net/fullchain.pem /tmp/ sudo cp /etc/letsencrypt/live/calypr-demo.ddns.net/privkey.pem /tmp/ sudo chmod 644 /tmp/fullchain.pem /tmp/privkey.pem - kubectl create secret tls ${TLS_SECRET_NAME} -n default --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n default --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n argocd --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n argo-workflows --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n argo-events --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n argo-stack --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n calypr-api --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true + kubectl create secret tls ${TLS_SECRET_NAME} -n calypr-tenants --cert=/tmp/fullchain.pem --key=/tmp/privkey.pem || true sudo rm /tmp/fullchain.pem /tmp/privkey.pem # install ingress helm upgrade --install ingress-authz-overlay \ @@ -184,14 +190,16 @@ ports: helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \ -n ingress-nginx --create-namespace \ --set controller.service.type=NodePort \ - --set controller.extraArgs.default-ssl-certificate=default/${TLS_SECRET_NAME} + --set controller.extraArgs.default-ssl-certificate=default/${TLS_SECRET_NAME} \ + --set controller.watchIngressWithoutClass=true \ + -f helm/argo-stack/overlays/ingress-authz-overlay/values-ingress-nginx.yaml # Assign external address (only if PUBLIC_IP is set) - @if [ -n "${PUBLIC_IP}" ]; then \ - echo "➡️ Assigning external IP: ${PUBLIC_IP}"; \ - kubectl patch svc ingress-nginx-controller -n ingress-nginx -p "{\"spec\": {\"type\": \"NodePort\", \"externalIPs\": [\"${PUBLIC_IP}\"]}}"; \ - else \ - echo "⚠️ PUBLIC_IP not set, skipping external IP assignment"; \ - fi + # @if [ -n "${PUBLIC_IP}" ]; then \ + # echo "➡️ Assigning external IP: ${PUBLIC_IP}"; \ + # kubectl patch svc ingress-nginx-controller -n ingress-nginx -p "{\"spec\": {\"type\": \"NodePort\", \"externalIPs\": [\"${PUBLIC_IP}\"]}}"; \ + # else \ + # echo "⚠️ PUBLIC_IP not set, skipping external IP assignment"; \ + # fi # Solution - Use NodePort instead of LoadBalancer in kind kubectl patch svc ingress-nginx-controller -n ingress-nginx -p '{"spec":{"type":"NodePort","ports":[{"port":80,"nodePort":30080},{"port":443,"nodePort":30443}]}}' diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml index f14933e6..1eb213e4 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/templates/ingress-authz.yaml @@ -70,7 +70,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-send-timeout: {{ $route.proxySendTimeout | quote }} {{- end }} spec: - ingressClassName: {{ $config.ingressClassName | default "nginx" | quote }} + ingressClassName: "nginx" {{- if $config.tls.enabled }} tls: - hosts: diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/values-ingress-nginx.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/values-ingress-nginx.yaml new file mode 100644 index 00000000..58dda035 --- /dev/null +++ b/helm/argo-stack/overlays/ingress-authz-overlay/values-ingress-nginx.yaml @@ -0,0 +1,4 @@ +controller: + config: + allow-snippet-annotations: "true" + annotations-risk-level: Critical # allow Critical-risk annotations like auth-snippet diff --git a/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml b/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml index 25117938..2b6bee17 100644 --- a/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml +++ b/helm/argo-stack/overlays/ingress-authz-overlay/values.yaml @@ -10,6 +10,7 @@ # --set ingressAuthzOverlay.enabled=true # ============================================================================ + ingressAuthzOverlay: # Enable or disable the overlay enabled: true @@ -102,7 +103,7 @@ ingressAuthzOverlay: # Only the primary route gets the cert-manager.io/cluster-issuer annotation primary: true # Namespace where the ingress will be created - namespace: argo-stack + namespace: argo-workflows # Service name to route to service: argo-stack-argo-workflows-server # Namespace where the actual service exists (for cross-namespace routing) @@ -118,7 +119,7 @@ ingressAuthzOverlay: # Argo CD Applications UI applications: enabled: true - namespace: argo-stack + namespace: argocd service: argo-stack-argocd-server # ArgoCD server is in argocd namespace serviceNamespace: argocd @@ -132,7 +133,7 @@ ingressAuthzOverlay: # GitHub Repository Registrations EventSource registrations: enabled: true - namespace: argo-stack + namespace: argo-events service: github-repo-registrations-eventsource-svc # EventSource is in argo-events namespace serviceNamespace: argo-events