Merge pull request #1 from stefanprodan/appmesh

Implement App Mesh service discovery

Makefile

@@ -7,6 +7,10 @@ DOCKER_IMAGE_NAME:=$(DOCKER_REPOSITORY)/$(NAME)
 run:
 	go run cmd/kxds/*.go kubernetes --kubeconfig=$$HOME/.kube/config --ads=true --port-name=http
 
+appmesh:
+	go run cmd/kxds/*.go appmesh --kubeconfig=$$HOME/.kube/config --ads=true \
+		--gateway-mesh=appmesh --gateway-name=gateway --gateway-namespace=appmesh-gateway
+
 envoy:
 	envoy -c envoy.yaml -l info
 

README.md

@@ -6,18 +6,27 @@ KxDS runs as a sidecar next to Envoy and configures the proxy to expose Kubernet
 
 ### Features
 
-* **Service Discovery** KxDS watches Kubernetes for ClusterIP services with a `http` named port
-* **Envoy Clusters (CDS)** are generated for each Kubernetes service in the form `<service-name>-<namespace>-<http-port>`
-* **Envoy Routes (RDS)** are generated for each cluster and mapped to the `<name>.<namespace>` domain
+* **Kubernetes Service Discovery** KxDS watches Kubernetes for services with a `http` named port
+* **App Mesh Service Discovery** KxDS watches Kubernetes for App Mesh virtual services
+* **Envoy Clusters (CDS)** are generated for each Kubernetes service or App Mesh virtual services
+* **Envoy Routes (RDS)** are generated for each cluster and configured with timeouts and retry policies
 * **Envoy Weighted Clusters** are generated based on Kubernetes service annotations
-* **Envoy Listeners (LDS)** KxDS configures Envoy to listen on port `8080` and sets up retry policies for each route
+* **Envoy Listeners (LDS)** KxDS configures Envoy to listen on port `8080`
 
 ### Install
 
+API Gateway for Kubernetes
+
 ```sh
 kubectl apply -k github.com/stefanprodan/kxds//kustomize/gateway
 ```
 
+API Gateway for App Mesh
+
+```sh
+kubectl apply -k github.com/stefanprodan/kxds//kustomize/appmesh-gateway
+```
+
 ### Annotations
 
 Kubernetes service:

cmd/kxds/appmesh.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+	"github.com/stefanprodan/kxds/pkg/discovery"
+	"k8s.io/client-go/dynamic"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+
+	"github.com/stefanprodan/kxds/pkg/envoy"
+	"github.com/stefanprodan/kxds/pkg/server"
+	"github.com/stefanprodan/kxds/pkg/signals"
+)
+
+var gatewayMesh string
+var gatewayName string
+var gatewayNamespace string
+
+func init() {
+	appmeshCmd.Flags().StringVarP(&masterURL, "master", "", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+	appmeshCmd.Flags().StringVarP(&kubeConfig, "kubeconfig", "", "", "Path to a kubeconfig. Only required if out-of-cluster.")
+	appmeshCmd.Flags().IntVarP(&port, "port", "p", 18000, "Port to listen on.")
+	appmeshCmd.Flags().StringVarP(&namespace, "namespace", "", "", "Namespace to watch for App Mesh virtual service.")
+	appmeshCmd.Flags().BoolVarP(&ads, "ads", "", true, "ADS flag forces a delay in responding to streaming requests until all resources are explicitly named in the request.")
+	appmeshCmd.Flags().StringVarP(&gatewayMesh, "gateway-mesh", "", "", "App Mesh gateway mesh.")
+	appmeshCmd.Flags().StringVarP(&gatewayName, "gateway-name", "", "", "App Mesh gateway name.")
+	appmeshCmd.Flags().StringVarP(&gatewayNamespace, "gateway-namespace", "", "", "App Mesh gateway namespace.")
+
+	rootCmd.AddCommand(appmeshCmd)
+}
+
+var appmeshCmd = &cobra.Command{
+	Use:   `appmesh`,
+	Short: "Starts App Mesh discovery service",
+	RunE:  appmeshRun,
+}
+
+func appmeshRun(cmd *cobra.Command, args []string) error {
+	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeConfig)
+	if err != nil {
+		klog.Fatalf("error building kubeconfig: %v", err)
+	}
+
+	clientset, err := dynamic.NewForConfig(cfg)
+	if err != nil {
+		klog.Fatalf("error building kubernetes clientset: %v", err)
+	}
+
+	stopCh := signals.SetupSignalHandler()
+	ctx := context.Background()
+	cache := envoy.NewCache(ads)
+	snapshot := envoy.NewSnapshot(cache)
+
+	klog.Infof("starting xDS server on port %d", port)
+	srv := server.NewServer(port, cache)
+	go srv.Serve(ctx)
+
+	klog.Info("waiting for Envoy to connect to the xDS server")
+	srv.Report()
+
+	klog.Info("starting App Mesh discovery workers")
+	kd := discovery.NewAppmeshDiscovery(clientset, namespace, snapshot, gatewayMesh, gatewayName, gatewayNamespace)
+	kd.Run(2, stopCh)
+
+	return nil
+}

cmd/kxds/version.go

@@ -6,7 +6,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var VERSION = "0.0.4"
+var VERSION = "0.1.0"
 
 func init() {
 	rootCmd.AddCommand(versionCmd)

kustomize/appmesh-gateway/deployment.yaml

@@ -0,0 +1,117 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: appmesh-gateway
+  labels:
+    app: appmesh-gateway
+spec:
+  selector:
+    matchLabels:
+      app: appmesh-gateway
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: appmesh-gateway
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/stats/prometheus"
+        prometheus.io/port: "8081"
+        # exclude inbound traffic on port 8080
+        appmesh.k8s.aws/ports: "444"
+        # exclude egress traffic to KxDS and Kubernetes API
+        appmesh.k8s.aws/egressIgnoredPorts: "18000,22,443"
+    spec:
+      serviceAccountName: appmesh-gateway
+      terminationGracePeriodSeconds: 10
+      containers:
+        - name: gateway
+          image: "envoyproxy/envoy:v1.11.1"
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+          command:
+            - /bin/sh
+          args:
+            - -c
+            - /docker-entrypoint.sh envoy --service-node ${POD_NAME} --service-cluster envoy --base-id 1234 -l info -c /config/envoy.yaml
+          ports:
+            - name: admin
+              containerPort: 8081
+              protocol: TCP
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            initialDelaySeconds: 5
+            tcpSocket:
+              port: admin
+          readinessProbe:
+            initialDelaySeconds: 5
+            tcpSocket:
+              port: admin
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+          volumeMounts:
+            - name: xds-config
+              mountPath: /config
+        - name: kxds
+          image: stefanprodan/kxds:0.1.0
+          imagePullPolicy: Always
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+          command:
+            - ./kxds
+            - appmesh
+            - --port=18000
+            - --ads=true
+            - --gateway-mesh=appmesh
+            - --gateway-name=$(POD_SERVICE_ACCOUNT)
+            - --gateway-namespace=$(POD_NAMESPACE)
+          env:
+            - name: POD_SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - name: grpc
+              containerPort: 18000
+              protocol: TCP
+          livenessProbe:
+            initialDelaySeconds: 5
+            tcpSocket:
+              port: grpc
+          readinessProbe:
+            initialDelaySeconds: 5
+            tcpSocket:
+              port: grpc
+      volumes:
+        - name: xds-config
+          configMap:
+            name: envoy

kustomize/appmesh-gateway/envoy.yaml

@@ -0,0 +1,33 @@
+admin:
+  access_log_path: /dev/stdout
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 8081
+
+dynamic_resources:
+  ads_config:
+    api_type: GRPC
+    grpc_services:
+      - envoy_grpc:
+          cluster_name: xds
+  cds_config:
+    ads: {}
+  lds_config:
+    ads: {}
+
+static_resources:
+  clusters:
+    - name: xds
+      connect_timeout: 0.30s
+      type: static
+      http2_protocol_options: {}
+      load_assignment:
+        cluster_name: xds
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: 127.0.0.1
+                      port_value: 18000

kustomize/appmesh-gateway/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: appmesh-gateway
+resources:
+- namespace.yaml
+- deployment.yaml
+- rbac.yaml
+- service.yaml
+configMapGenerator:
+  - name: envoy
+    files:
+      - envoy.yaml

kustomize/appmesh-gateway/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: appmesh-gateway
+  annotations:
+    fluxcd.io/ignore: "false"
+  labels:
+    appmesh.k8s.aws/sidecarInjectorWebhook: enabled

kustomize/appmesh-gateway/rbac.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: appmesh-gateway
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: appmesh-gateway
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs: ["*"]
+  - apiGroups:
+      - appmesh.k8s.aws
+    resources:
+      - meshes
+      - meshes/status
+      - virtualnodes
+      - virtualnodes/status
+      - virtualservices
+      - virtualservices/status
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: appmesh-gateway
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: appmesh-gateway
+subjects:
+- kind: ServiceAccount
+  name: appmesh-gateway
+  namespace: appmesh-gateway

kustomize/appmesh-gateway/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: appmesh-gateway
+  labels:
+    app: appmesh-gateway
+  annotations:
+    envoy.gateway.kubernetes.io/expose: "false"
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+  selector:
+    app: appmesh-gateway