diff --git a/.github/actions/aqua/action.yaml b/.github/actions/aqua/action.yaml index 88841a5..de052a4 100644 --- a/.github/actions/aqua/action.yaml +++ b/.github/actions/aqua/action.yaml @@ -7,8 +7,8 @@ inputs: runs: using: composite steps: - - uses: aquaproj/aqua-installer@36dc5833b04eb63f06e3bb818aa6b7a6e6db99a9 # v2.1.2 + - uses: aquaproj/aqua-installer@7c7338067bdb97d5bea2acc82b5870afca470d18 # v2.3.0 with: - aqua_version: v2.9.0 + aqua_version: v2.22.0 env: GITHUB_TOKEN: ${{ inputs.github_token }} diff --git a/README.md b/README.md index 48f71eb..d74077b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Cattage is a Kubernetes controller that enhances the multi-tenancy of [Argo CD][ - Management of root-namespaces for tenants. Tenant users will be able to create sub-namespaces in those root-namespaces. - When a tenant user creates a sub-namespace, the AppProject will be automatically updated accordingly. Tenant users will be able to deploy applications with Argo CD to the namespaces. - The ownership of sub-namespaces can be changed between tenants. +- Sharding application-controller instances. ## Supported Version diff --git a/Tiltfile b/Tiltfile index be9750f..679af90 100644 --- a/Tiltfile +++ b/Tiltfile @@ -23,7 +23,6 @@ watch_file('./config/') k8s_yaml(kustomize('./config/dev')) k8s_resource(new_name='Cattage Resources', objects=[ 'cattage:namespace', - 'tenants.cattage.cybozu.io:customresourcedefinition', 'cattage-mutating-webhook-configuration:mutatingwebhookconfiguration', 'cattage-controller-manager:serviceaccount', 'cattage-leader-election-role:role', diff --git a/api/v1beta1/tenant_types.go b/api/v1beta1/tenant_types.go index 1f9bee3..8c28ff6 100644 --- a/api/v1beta1/tenant_types.go +++ b/api/v1beta1/tenant_types.go @@ -18,6 +18,11 @@ type TenantSpec struct { // Delegates is a list of other tenants that are delegated access to this tenant. // +optional Delegates []DelegateSpec `json:"delegates,omitempty"` + + // ControllerName is the name of the application-controller that manages this tenant's applications. + // If not specified, the default controller is used. + // +optional + ControllerName string `json:"controllerName,omitempty"` } // RootNamespaceSpec defines the desired state of Namespace. diff --git a/charts/cattage/Chart.yaml b/charts/cattage/Chart.yaml index 9ddf83f..b3b5aa8 100644 --- a/charts/cattage/Chart.yaml +++ b/charts/cattage/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0-chart-patch-version-placeholder +version: 0.6.0-chart-patch-version-placeholder # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/cattage/crds/tenant.yaml b/charts/cattage/crds/tenant.yaml index 57c5f93..ce094e8 100644 --- a/charts/cattage/crds/tenant.yaml +++ b/charts/cattage/crds/tenant.yaml @@ -53,6 +53,11 @@ spec: type: string type: array type: object + controllerName: + description: |- + ControllerName is the name of the application-controller that manages this tenant's applications. + If not specified, the default controller is used. + type: string delegates: description: Delegates is a list of other tenants that are delegated access to this tenant. items: diff --git a/charts/cattage/templates/configmap.yaml b/charts/cattage/templates/configmap.yaml index 68306b1..321aca2 100644 --- a/charts/cattage/templates/configmap.yaml +++ b/charts/cattage/templates/configmap.yaml @@ -18,3 +18,4 @@ data: argocd: namespace: {{ required ".Values.controller.config.argocd.namespace required!" .Values.controller.config.argocd.namespace }} appProjectTemplate: {{ required ".Values.controller.config.argocd.appProjectTemplate required!" .Values.controller.config.argocd.appProjectTemplate | toYaml | nindent 8 }} + preventAppCreationInArgoCDNamespace: {{ required ".Values.controller.config.argocd.preventAppCreationInArgoCDNamespace required!" .Values.controller.config.argocd.preventAppCreationInArgoCDNamespace }} diff --git a/charts/cattage/templates/generated.yaml b/charts/cattage/templates/generated.yaml index c95a015..adfda26 100644 --- a/charts/cattage/templates/generated.yaml +++ b/charts/cattage/templates/generated.yaml @@ -62,6 +62,18 @@ metadata: helm.sh/chart: '{{ include "cattage.chart" . }}' name: '{{ template "cattage.fullname" . }}-manager-role' rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -282,6 +294,26 @@ metadata: helm.sh/chart: '{{ include "cattage.chart" . }}' name: '{{ template "cattage.fullname" . }}-validating-webhook-configuration' webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ template "cattage.fullname" . }}-webhook-service' + namespace: '{{ .Release.Namespace }}' + path: /validate-argoproj-io-application + failurePolicy: Fail + name: vapplication.kb.io + rules: + - apiGroups: + - argoproj.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - applications + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/charts/cattage/values.yaml b/charts/cattage/values.yaml index 838522e..b67c014 100644 --- a/charts/cattage/values.yaml +++ b/charts/cattage/values.yaml @@ -81,3 +81,4 @@ controller: {{- else }} - '*' {{- end }} + preventAppCreationInArgoCDNamespace: false diff --git a/cmd/cattage-controller/sub/run.go b/cmd/cattage-controller/sub/run.go index abe454a..07fa434 100644 --- a/cmd/cattage-controller/sub/run.go +++ b/cmd/cattage-controller/sub/run.go @@ -82,6 +82,7 @@ func subMain(ns, addr string, port int) error { } hooks.SetupTenantWebhook(mgr, admission.NewDecoder(scheme), cfg) + hooks.SetupApplicationWebhook(mgr, admission.NewDecoder(scheme), cfg) //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/config/crd/bases/cattage.cybozu.io_tenants.yaml b/config/crd/bases/cattage.cybozu.io_tenants.yaml index 23b788b..6c9dc30 100644 --- a/config/crd/bases/cattage.cybozu.io_tenants.yaml +++ b/config/crd/bases/cattage.cybozu.io_tenants.yaml @@ -53,6 +53,11 @@ spec: type: string type: array type: object + controllerName: + description: |- + ControllerName is the name of the application-controller that manages this tenant's applications. + If not specified, the default controller is used. + type: string delegates: description: Delegates is a list of other tenants that are delegated access to this tenant. diff --git a/config/manager/configmap.yaml b/config/manager/configmap.yaml index e65b1ff..63f7b59 100644 --- a/config/manager/configmap.yaml +++ b/config/manager/configmap.yaml @@ -65,3 +65,4 @@ data: {{- else }} - '*' {{- end }} + preventAppCreationInArgoCDNamespace: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index fe802a3..4ba9b7b 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/config/samples/tenant.yaml b/config/samples/tenant.yaml index 98cee88..bb85a3f 100644 --- a/config/samples/tenant.yaml +++ b/config/samples/tenant.yaml @@ -5,6 +5,7 @@ metadata: spec: rootNamespaces: - name: app-a + controllerName: second --- apiVersion: cattage.cybozu.io/v1beta1 kind: Tenant diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 2d194c1..34f9589 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -30,6 +30,26 @@ kind: ValidatingWebhookConfiguration metadata: name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-argoproj-io-application + failurePolicy: Fail + name: vapplication.kb.io + rules: + - apiGroups: + - argoproj.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - applications + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index 1bbefaf..22d26d2 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "strings" "text/template" cattagev1beta1 "github.com/cybozu-go/cattage/api/v1beta1" @@ -57,6 +58,7 @@ type TenantReconciler struct { //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles,verbs=get;list;watch;escalate;bind //+kubebuilder:rbac:groups=argoproj.io,resources=applications,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -124,6 +126,18 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return ctrl.Result{}, err } + err = r.reconcileConfigMapForApplicationController(ctx, tenant) + if err != nil { + tenant.Status.Health = cattagev1beta1.TenantUnhealthy + meta.SetStatusCondition(&tenant.Status.Conditions, metav1.Condition{ + Type: cattagev1beta1.ConditionReady, + Status: metav1.ConditionFalse, + Reason: "Failed", + Message: err.Error(), + }) + return ctrl.Result{}, err + } + tenant.Status.Health = cattagev1beta1.TenantHealthy meta.SetStatusCondition(&tenant.Status.Conditions, metav1.Condition{ Type: cattagev1beta1.ConditionReady, @@ -252,7 +266,7 @@ func (r *TenantReconciler) finalize(ctx context.Context, tenant *cattagev1beta1. } logger.Info("starting finalization") nss := &corev1.NamespaceList{} - if err := r.client.List(ctx, nss, client.MatchingFields{constants.RootNamespaces: tenant.Name}); err != nil { + if err := r.client.List(ctx, nss, client.MatchingFields{constants.RootNamespaceIndex: tenant.Name}); err != nil { return fmt.Errorf("failed to list namespaces: %w", err) } for _, ns := range nss.Items { @@ -411,7 +425,7 @@ func (r *TenantReconciler) reconcileNamespaces(ctx context.Context, tenant *catt } } nss := &corev1.NamespaceList{} - if err := r.client.List(ctx, nss, client.MatchingFields{constants.RootNamespaces: tenant.Name}); err != nil { + if err := r.client.List(ctx, nss, client.MatchingFields{constants.RootNamespaceIndex: tenant.Name}); err != nil { return fmt.Errorf("failed to list namespaces: %w", err) } for _, ns := range nss.Items { @@ -442,7 +456,7 @@ func (r *TenantReconciler) reconcileArgoCD(ctx context.Context, tenant *cattagev } nss := &corev1.NamespaceList{} - if err := r.client.List(ctx, nss, client.MatchingFields{constants.TenantNamespaces: tenant.Name}); err != nil { + if err := r.client.List(ctx, nss, client.MatchingFields{constants.TenantNamespaceIndex: tenant.Name}); err != nil { return fmt.Errorf("failed to list namespaces: %w", err) } namespaces := make([]string, len(nss.Items)) @@ -505,6 +519,150 @@ func (r *TenantReconciler) reconcileArgoCD(ctx context.Context, tenant *cattagev return nil } +func (r *TenantReconciler) reconcileConfigMapForApplicationController(ctx context.Context, tenant *cattagev1beta1.Tenant) error { + cmList := &corev1.ConfigMapList{} + err := r.client.List(ctx, cmList, client.MatchingLabels{constants.ManagedByLabel: "cattage"}) + if err != nil { + return err + } + controllerNames := map[string]struct{}{} + for _, cm := range cmList.Items { + if cm.Labels[constants.ControllerNameLabel] != "" { + controllerNames[cm.Labels[constants.ControllerNameLabel]] = struct{}{} + } + } + controllerName := tenant.Spec.ControllerName + if controllerName == "" { + controllerName = constants.DefaultApplicationControllerName + } + controllerNames[controllerName] = struct{}{} + + for name := range controllerNames { + err := r.updateConfigMap(ctx, name) + if err != nil { + return err + } + } + + err = r.updateAllTenantNamespacesConfigMap(ctx) + if err != nil { + return err + } + + return nil +} + +func (r *TenantReconciler) updateConfigMap(ctx context.Context, controllerName string) error { + logger := log.FromContext(ctx) + + configMapName := controllerName + "-application-controller-cm" + cm := &corev1.ConfigMap{} + cm.Name = configMapName + cm.Namespace = r.config.ArgoCD.Namespace + + tenants := &cattagev1beta1.TenantList{} + if err := r.client.List(ctx, tenants, client.MatchingFields{constants.ControllerNameIndex: controllerName}); err != nil { + return fmt.Errorf("failed to list tenants: %w", err) + } + + if len(tenants.Items) == 0 { + err := r.client.Delete(ctx, cm) + return err + } + + namespaces := make([]string, 0) + for _, t := range tenants.Items { + nss := &corev1.NamespaceList{} + if err := r.client.List(ctx, nss, client.MatchingFields{constants.TenantNamespaceIndex: t.Name}); err != nil { + return fmt.Errorf("failed to list namespaces: %w", err) + } + for _, ns := range nss.Items { + namespaces = append(namespaces, ns.Name) + } + } + + op, err := ctrl.CreateOrUpdate(ctx, r.client, cm, func() error { + cm.Labels = map[string]string{ + constants.ManagedByLabel: "cattage", + constants.PartOfLabel: "argocd", + constants.ControllerNameLabel: controllerName, + } + cm.Data = map[string]string{ + "application.namespaces": strings.Join(namespaces, ","), + } + cm.OwnerReferences = nil + for _, tenant := range tenants.Items { + err := controllerutil.SetOwnerReference(&tenant, cm, r.client.Scheme()) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + logger.Error(err, "failed to update ConfigMap") + return err + } + if op != controllerutil.OperationResultNone { + logger.Info("ConfigMap successfully reconciled") + } + + return nil +} + +func (r *TenantReconciler) updateAllTenantNamespacesConfigMap(ctx context.Context) error { + logger := log.FromContext(ctx) + + configMapName := "all-tenant-namespaces-cm" + cm := &corev1.ConfigMap{} + cm.Name = configMapName + cm.Namespace = r.config.ArgoCD.Namespace + + tenantList := &cattagev1beta1.TenantList{} + err := r.client.List(ctx, tenantList) + if err != nil { + return err + } + + allNamespaces := make([]string, 0) + for _, tenant := range tenantList.Items { + nss := &corev1.NamespaceList{} + if err := r.client.List(ctx, nss, client.MatchingFields{constants.TenantNamespaceIndex: tenant.Name}); err != nil { + return fmt.Errorf("failed to list namespaces: %w", err) + } + for _, ns := range nss.Items { + allNamespaces = append(allNamespaces, ns.Name) + } + } + + op, err := ctrl.CreateOrUpdate(ctx, r.client, cm, func() error { + cm.Labels = map[string]string{ + constants.ManagedByLabel: "cattage", + constants.PartOfLabel: "argocd", + } + cm.Data = map[string]string{ + "application.namespaces": strings.Join(allNamespaces, ","), + } + cm.OwnerReferences = nil + for _, tenant := range tenantList.Items { + err := controllerutil.SetOwnerReference(&tenant, cm, r.client.Scheme()) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + logger.Error(err, "failed to update ConfigMap") + return err + } + if op != controllerutil.OperationResultNone { + logger.Info("ConfigMap successfully reconciled") + } + + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error { tenantHandler := func(ctx context.Context, o client.Object) []reconcile.Request { @@ -525,7 +683,7 @@ func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error { func SetupIndexForNamespace(ctx context.Context, mgr manager.Manager) error { ns := &corev1.Namespace{} - err := mgr.GetFieldIndexer().IndexField(ctx, ns, constants.RootNamespaces, func(rawObj client.Object) []string { + err := mgr.GetFieldIndexer().IndexField(ctx, ns, constants.RootNamespaceIndex, func(rawObj client.Object) []string { nsType := rawObj.GetLabels()[accurate.LabelType] if nsType != accurate.NSTypeRoot { return nil @@ -540,11 +698,24 @@ func SetupIndexForNamespace(ctx context.Context, mgr manager.Manager) error { return err } - return mgr.GetFieldIndexer().IndexField(ctx, ns, constants.TenantNamespaces, func(rawObj client.Object) []string { + err = mgr.GetFieldIndexer().IndexField(ctx, ns, constants.TenantNamespaceIndex, func(rawObj client.Object) []string { tenantName := rawObj.GetLabels()[constants.OwnerTenant] if tenantName == "" { return nil } return []string{tenantName} }) + if err != nil { + return err + } + + tenant := &cattagev1beta1.Tenant{} + return mgr.GetFieldIndexer().IndexField(ctx, tenant, constants.ControllerNameIndex, func(rawObj client.Object) []string { + tenant := rawObj.(*cattagev1beta1.Tenant) + controllerName := tenant.Spec.ControllerName + if controllerName == "" { + return []string{constants.DefaultApplicationControllerName} + } + return []string{controllerName} + }) } diff --git a/controllers/tenant_controller_test.go b/controllers/tenant_controller_test.go index 89aad58..9aa841c 100644 --- a/controllers/tenant_controller_test.go +++ b/controllers/tenant_controller_test.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "errors" + "strings" "time" cattagev1beta1 "github.com/cybozu-go/cattage/api/v1beta1" @@ -30,7 +31,7 @@ var appProjectTemplate string //go:embed testdata/rolebindingtemplate.yaml var roleBindingTemplate string -var _ = Describe("Tenant controller", func() { +var _ = Describe("Tenant controller", Ordered, func() { ctx := context.Background() var stopFunc func() var config *tenantconfig.Config @@ -61,8 +62,9 @@ var _ = Describe("Tenant controller", func() { RoleBindingTemplate: roleBindingTemplate, }, ArgoCD: tenantconfig.ArgoCDConfig{ - Namespace: "argocd", - AppProjectTemplate: appProjectTemplate, + Namespace: "argocd", + AppProjectTemplate: appProjectTemplate, + PreventAppCreationInArgoCDNamespace: true, }, } tr := NewTenantReconciler(mgr.GetClient(), config) @@ -205,6 +207,60 @@ var _ = Describe("Tenant controller", func() { })) }) + It("should create configmaps for sharding", func() { + allNsCm := &corev1.ConfigMap{} + defaultCm := &corev1.ConfigMap{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "all-tenant-namespaces-cm"}, allNsCm) + g.Expect(err).NotTo(HaveOccurred()) + allNs := strings.Split(allNsCm.Data["application.namespaces"], ",") + g.Expect(allNs).Should(ConsistOf("app-x", "sub-4")) + }).Should(Succeed()) + + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "default-application-controller-cm"}, defaultCm) + g.Expect(err).NotTo(HaveOccurred()) + defaultNs := strings.Split(defaultCm.Data["application.namespaces"], ",") + g.Expect(defaultNs).Should(ConsistOf("app-x", "sub-4")) + }).Should(Succeed()) + + tenantS := &cattagev1beta1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-team", + }, + Spec: cattagev1beta1.TenantSpec{ + RootNamespaces: []cattagev1beta1.RootNamespaceSpec{ + { + Name: "app-a", + }, + }, + ControllerName: "second", + }, + } + err := k8sClient.Create(ctx, tenantS) + Expect(err).ToNot(HaveOccurred()) + + secondCm := &corev1.ConfigMap{} + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "all-tenant-namespaces-cm"}, allNsCm) + g.Expect(err).NotTo(HaveOccurred()) + allNs := strings.Split(allNsCm.Data["application.namespaces"], ",") + g.Expect(allNs).Should(ConsistOf("app-x", "sub-4", "app-a", "sub-1", "sub-2", "sub-3")) + }).Should(Succeed()) + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "default-application-controller-cm"}, defaultCm) + g.Expect(err).NotTo(HaveOccurred()) + defaultNs := strings.Split(defaultCm.Data["application.namespaces"], ",") + g.Expect(defaultNs).Should(ConsistOf("app-x", "sub-4")) + }).Should(Succeed()) + Eventually(func(g Gomega) { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "argocd", Name: "second-application-controller-cm"}, secondCm) + g.Expect(err).NotTo(HaveOccurred()) + secondNs := strings.Split(secondCm.Data["application.namespaces"], ",") + g.Expect(secondNs).Should(ConsistOf("app-a", "sub-1", "sub-2", "sub-3")) + }).Should(Succeed()) + }) + It("should disown root namespace", func() { tenant := &cattagev1beta1.Tenant{ ObjectMeta: metav1.ObjectMeta{ diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index fb743f3..0a44192 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -7,6 +7,7 @@ - [Overview](overview.md) - [Setup](setup.md) - [Usage](usage.md) +- [Sharding](sharding.md) # References diff --git a/docs/config.md b/docs/config.md index a353281..bdd504a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -7,13 +7,14 @@ The location can be changed with `--config-file` flag. The configuration file should be a JSON or YAML file having the following keys: -| Key | Type | Description | -|---------------------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| `namespace.commonLabels` | `map[string]string` | Labels to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.labels` of a tenant resource. | -| `namespace.commonAnnotations` | `map[string]string` | Annotations to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.annotations` of a tenant resource. | -| `namespace.roleBindingTemplate` | `string` | Template for RoleBinding resource that is created on all namespaces belonging to a tenant. | -| `argocd.namepsace` | `string` | The name of namespace where Argo CD is running. | -| `argocd.appProjectTemplate` | `string` | Template for AppProject resources that is created for each tenant. | +| Key | Type | Description | +|----------------------------------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| `namespace.commonLabels` | `map[string]string` | Labels to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.labels` of a tenant resource. | +| `namespace.commonAnnotations` | `map[string]string` | Annotations to be added to all namespaces belonging to all tenants. This may be overridden by `rootNamespaces.annotations` of a tenant resource. | +| `namespace.roleBindingTemplate` | `string` | Template for RoleBinding resource that is created on all namespaces belonging to a tenant. | +| `argocd.namepsace` | `string` | The name of namespace where Argo CD is running. | +| `argocd.appProjectTemplate` | `string` | Template for AppProject resources that is created for each tenant. | +| `argocd.preventAppCreationInArgoCDNamespace` | `bool` | If true, prevent creating applications in the Argo CD namespace. This is used to enable sharding. | The repository includes an example as follows: diff --git a/docs/crd_tenant.md b/docs/crd_tenant.md index 52ab2ea..2705dcb 100644 --- a/docs/crd_tenant.md +++ b/docs/crd_tenant.md @@ -77,6 +77,7 @@ TenantSpec defines the desired state of Tenant. | rootNamespaces | RootNamespaces are the list of root namespaces that belong to this tenant. | [][RootNamespaceSpec](#rootnamespacespec) | true | | argocd | ArgoCD is the settings of Argo CD for this tenant. | [ArgoCDSpec](#argocdspec) | false | | delegates | Delegates is a list of other tenants that are delegated access to this tenant. | [][DelegateSpec](#delegatespec) | false | +| controllerName | ControllerName is the name of the application-controller that manages this tenant's applications. If not specified, the default controller is used. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/overview.md b/docs/overview.md index 772b9b2..35a036b 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -21,6 +21,11 @@ Cattage is a Kubernetes controller that enhances the multi-tenancy of [Argo CD][ Sometimes users may want to move the ownership of an application to another tenant. When the parent of a sub-namespace is changed, Cattage will automatically update the permissions. +- Sharding application-controller instances + + Cattage can shard application-controller instances by the tenant. + This feature is useful when you have a large number of tenants and want to avoid a single application-controller instance from being overloaded. + [Accurate]: https://github.com/cybozu-go/accurate [Argo CD]: https://argo-cd.readthedocs.io/en/stable/ [App Of Apps Pattern]: https://argo-cd.readthedocs.io/en/stable/operator-manual/cluster-bootstrapping/#app-of-apps-pattern diff --git a/docs/sharding.md b/docs/sharding.md new file mode 100644 index 0000000..176c380 --- /dev/null +++ b/docs/sharding.md @@ -0,0 +1,155 @@ +# Sharding + +## Overview + +In Argo CD, as the number of managed applications increases, the load on the Application Controller becomes significant. +While Argo CD supports sharding, it can only shard controllers per Kubernetes cluster. (ref. https://argo-cd.readthedocs.io/en/stable/operator-manual/high_availability/ ) + +Cattage provides the capability to shard controllers on a per-tenant basis using [applications in any namespace](https://argo-cd.readthedocs.io/en/stable/operator-manual/app-any-namespace/). +By specifying a controller name in the Tenant resource, you can designate which controller will process Applications created in that tenant's Namespaces. + +## How to use + +### Setup stakater/Reloader + +[stakater/Reloader](https://github.com/stakater/Reloader) is a Kubernetes controller that watches for changes in ConfigMaps and Secrets, executing rolling updates on Deployments and StatefulSets as needed. +Cattage uses `stakater/Reloader` to roll out updates to the Argo CD Application Controller whenever a ConfigMap is modified. + +Follow these steps to set it up: + +```bash +helm repo add stakater https://stakater.github.io/stakater-charts +helm repo update +helm install --create-namespace --namespace reloader reloader -f manifests/reloader-values.yaml stakater/reloader +``` + +### Setup ArgoCD + +Set up Argo CD with the following commands: + +```bash +helm repo add argo https://argoproj.github.io/argo-helm +helm repo update +helm install --create-namespace --namespace argocd argocd argo/argo-cd +``` + +Copy and rename the StatefulSet for the Application Controller, then deploy it. +Repeat for as many controllers as required for sharding: + +```bash +helm template argo/argo-cd | yq ea '. as $$i ireduce ([]; . + $$i) | .[] | select(.kind=="StatefulSet") | .metadata.name="second-application-controller"' | kubectl apply -f - +``` + +Apply patches to the Application Controllers, ArgoCD Server, and Notification Controller to use the ConfigMaps generated by Cattage, and add an annotation to enable `stakater/Reloader`. + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: argocd-application-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: default-application-controller-cm + optional: true + name: application-controller +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: second-application-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: second-application-controller-cm + optional: true + name: application-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-notifications-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: all-tenant-namespaces-cm + optional: true + name: notifications-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-server + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: all-tenant-namespaces-cm + optional: true + name: server +``` + +Cattage generates the following configmaps: + +- `all-tenant-namespaces-cm`: Lists namespaces belonging to all tenants +- `default-application-controller-cm`: Lists namespaces for tenants without a specified controller +- `-application-controller-cm`: Lists namespaces for tenants with a specified controller + +### Setup Cattage + +Follow the [setup instructions](./setup.md) to install Cattage. + +Ensure `controller.config.argocd.preventAppCreationInArgoCDNamespace` in values.yaml is enabled to avoid multiple controllers processing the same Application in argocd namespace. + +### Creating Tenant Resources + +When creating a Tenant resource, specify the controllerName. + +```yaml +apiVersion: cattage.cybozu.io/v1beta1 +kind: Tenant +metadata: + name: a-team +spec: + rootNamespaces: + - name: app-a + controllerName: second +``` + +Applications created in the Namespace of that tenant will then be processed by the specified application controller. +If no controller name is specified, it will be processed by the default application controller. diff --git a/e2e/Makefile b/e2e/Makefile index 7097450..09c7649 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -24,19 +24,31 @@ start: .PHONY: prepare prepare: + # Setup cert-manager kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml kubectl -n cert-manager wait --for=condition=available --timeout=180s --all deployments - helm repo add argo https://argoproj.github.io/argo-helm + # Setup stakater/Reloader + helm repo add stakater https://stakater.github.io/stakater-charts helm repo update - helm install --create-namespace --namespace argocd argocd -f argocd-values.yaml argo/argo-cd + helm install --create-namespace --namespace reloader reloader -f manifests/reloader-values.yaml stakater/reloader + kubectl -n reloader wait --for=condition=available --timeout=180s --all deployments + + # Setup Argo CD + kubectl create namespace argocd + kustomize build --enable-helm ./manifests | kubectl apply -f - kubectl -n argocd wait --for=condition=available --timeout=180s --all deployments + # Setup accurate helm repo add accurate https://cybozu-go.github.io/accurate helm repo update - helm install --create-namespace --namespace accurate accurate -f accurate-values.yaml accurate/accurate + helm install --create-namespace --namespace accurate accurate -f manifests/accurate-values.yaml accurate/accurate kubectl -n accurate wait --for=condition=available --timeout=180s --all deployments +.PHONY: generate-second-controller +generate-second-controller: + kustomize build --enable-helm ./manifests | yq ea '. as $$i ireduce ([]; . + $$i) | .[] | select(.kind=="StatefulSet") | .metadata.name="second-application-controller"' > ./manifests/second-application-controller.yaml + .PHONY: test test: env RUN_E2E=1 \ diff --git a/e2e/argocd-values.yaml b/e2e/argocd-values.yaml deleted file mode 100644 index 0ad2562..0000000 --- a/e2e/argocd-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -global: - # Default image used by all components - image: - # -- Overrides the global Argo CD image tag whose default is the chart appVersion - tag: "v2.10.2" -## Argo Configs -configs: - # Argo CD configuration parameters - ## Ref: https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/argocd-cmd-params-cm.yaml - params: - # -- Enables [Applications in any namespace] - ## List of additional namespaces where applications may be created in and reconciled from. - ## The namespace where Argo CD is installed to will always be allowed. - ## Set comma-separated list. (e.g. app-team-one, app-team-two) - application.namespaces: "*" diff --git a/e2e/manifests/.gitignore b/e2e/manifests/.gitignore new file mode 100644 index 0000000..ebf1d3d --- /dev/null +++ b/e2e/manifests/.gitignore @@ -0,0 +1 @@ +charts diff --git a/e2e/accurate-values.yaml b/e2e/manifests/accurate-values.yaml similarity index 100% rename from e2e/accurate-values.yaml rename to e2e/manifests/accurate-values.yaml diff --git a/e2e/manifests/argocd-patches.yaml b/e2e/manifests/argocd-patches.yaml new file mode 100644 index 0000000..6babb52 --- /dev/null +++ b/e2e/manifests/argocd-patches.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: argocd-application-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: default-application-controller-cm + optional: true + name: application-controller +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: second-application-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: second-application-controller-cm + optional: true + name: application-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-notifications-controller + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: all-tenant-namespaces-cm + optional: true + name: notifications-controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-server + namespace: argocd + annotations: + reloader.stakater.com/auto: "true" +spec: + template: + spec: + containers: + - env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: all-tenant-namespaces-cm + optional: true + name: server diff --git a/e2e/manifests/argocd-values.yaml b/e2e/manifests/argocd-values.yaml new file mode 100644 index 0000000..c86af8d --- /dev/null +++ b/e2e/manifests/argocd-values.yaml @@ -0,0 +1,5 @@ +global: + # Default image used by all components + image: + # -- Overrides the global Argo CD image tag whose default is the chart appVersion + tag: "v2.10.2" diff --git a/e2e/manifests/kustomization.yaml b/e2e/manifests/kustomization.yaml new file mode 100644 index 0000000..bf4c9be --- /dev/null +++ b/e2e/manifests/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +helmCharts: +- includeCRDs: true + name: argo-cd + namespace: argocd + releaseName: argocd + repo: https://argoproj.github.io/argo-helm + valuesFile: argocd-values.yaml + version: 6.6.0 +namespace: argocd +resources: + - second-application-controller.yaml +patches: + - path: argocd-patches.yaml diff --git a/e2e/manifests/reloader-values.yaml b/e2e/manifests/reloader-values.yaml new file mode 100644 index 0000000..9f6269c --- /dev/null +++ b/e2e/manifests/reloader-values.yaml @@ -0,0 +1,2 @@ +reloader: + reloadOnCreate: true diff --git a/e2e/manifests/second-application-controller.yaml b/e2e/manifests/second-application-controller.yaml new file mode 100644 index 0000000..b725cd5 --- /dev/null +++ b/e2e/manifests/second-application-controller.yaml @@ -0,0 +1,288 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: application-controller + app.kubernetes.io/instance: argocd + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: argocd-application-controller + app.kubernetes.io/part-of: argocd + app.kubernetes.io/version: v2.10.2 + helm.sh/chart: argo-cd-6.6.0 + name: second-application-controller + namespace: argocd +spec: + replicas: 1 + revisionHistoryLimit: 5 + selector: + matchLabels: + app.kubernetes.io/instance: argocd + app.kubernetes.io/name: argocd-application-controller + serviceName: argocd-application-controller + template: + metadata: + annotations: + checksum/cm: d02c4fe2f8ec703f51ffb28b8713e71486ab241140206fa2db767c6b99a013e6 + checksum/cmd-params: bfd9e34756e1e442a08fef6509098fda42b19b64f44f8eb63c963516472f9442 + labels: + app.kubernetes.io/component: application-controller + app.kubernetes.io/instance: argocd + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: argocd-application-controller + app.kubernetes.io/part-of: argocd + app.kubernetes.io/version: v2.10.2 + helm.sh/chart: argo-cd-6.6.0 + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: argocd-application-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - /usr/local/bin/argocd-application-controller + - --metrics-port=8082 + env: + - name: ARGOCD_APPLICATION_NAMESPACES + valueFrom: + configMapKeyRef: + key: application.namespaces + name: default-application-controller-cm + optional: true + - name: ARGOCD_CONTROLLER_REPLICAS + value: "1" + - name: ARGOCD_APPLICATION_CONTROLLER_NAME + value: argocd-application-controller + - name: ARGOCD_RECONCILIATION_TIMEOUT + valueFrom: + configMapKeyRef: + key: timeout.reconciliation + name: argocd-cm + optional: true + - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT + valueFrom: + configMapKeyRef: + key: timeout.hard.reconciliation + name: argocd-cm + optional: true + - name: ARGOCD_RECONCILIATION_JITTER + valueFrom: + configMapKeyRef: + key: timeout.reconciliation.jitter + name: argocd-cm + optional: true + - name: ARGOCD_REPO_ERROR_GRACE_PERIOD_SECONDS + valueFrom: + configMapKeyRef: + key: controller.repo.error.grace.period.seconds + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER + valueFrom: + configMapKeyRef: + key: repo.server + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS + valueFrom: + configMapKeyRef: + key: controller.repo.server.timeout.seconds + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_STATUS_PROCESSORS + valueFrom: + configMapKeyRef: + key: controller.status.processors + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_OPERATION_PROCESSORS + valueFrom: + configMapKeyRef: + key: controller.operation.processors + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_LOGFORMAT + valueFrom: + configMapKeyRef: + key: controller.log.format + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_LOGLEVEL + valueFrom: + configMapKeyRef: + key: controller.log.level + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_METRICS_CACHE_EXPIRATION + valueFrom: + configMapKeyRef: + key: controller.metrics.cache.expiration + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_SELF_HEAL_TIMEOUT_SECONDS + valueFrom: + configMapKeyRef: + key: controller.self.heal.timeout.seconds + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_PLAINTEXT + valueFrom: + configMapKeyRef: + key: controller.repo.server.plaintext + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_STRICT_TLS + valueFrom: + configMapKeyRef: + key: controller.repo.server.strict.tls + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_PERSIST_RESOURCE_HEALTH + valueFrom: + configMapKeyRef: + key: controller.resource.health.persist + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APP_STATE_CACHE_EXPIRATION + valueFrom: + configMapKeyRef: + key: controller.app.state.cache.expiration + name: argocd-cmd-params-cm + optional: true + - name: REDIS_SERVER + valueFrom: + configMapKeyRef: + key: redis.server + name: argocd-cmd-params-cm + optional: true + - name: REDIS_COMPRESSION + valueFrom: + configMapKeyRef: + key: redis.compression + name: argocd-cmd-params-cm + optional: true + - name: REDISDB + valueFrom: + configMapKeyRef: + key: redis.db + name: argocd-cmd-params-cm + optional: true + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + key: redis-username + name: argocd-redis + optional: true + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + key: redis-password + name: argocd-redis + optional: true + - name: ARGOCD_DEFAULT_CACHE_EXPIRATION + valueFrom: + configMapKeyRef: + key: controller.default.cache.expiration + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_ADDRESS + valueFrom: + configMapKeyRef: + key: otlp.address + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_INSECURE + valueFrom: + configMapKeyRef: + key: otlp.insecure + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_OTLP_HEADERS + valueFrom: + configMapKeyRef: + key: otlp.headers + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_CONTROLLER_SHARDING_ALGORITHM + valueFrom: + configMapKeyRef: + key: controller.sharding.algorithm + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_KUBECTL_PARALLELISM_LIMIT + valueFrom: + configMapKeyRef: + key: controller.kubectl.parallelism.limit + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_K8SCLIENT_RETRY_MAX + valueFrom: + configMapKeyRef: + key: controller.k8sclient.retry.max + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_K8SCLIENT_RETRY_BASE_BACKOFF + valueFrom: + configMapKeyRef: + key: controller.k8sclient.retry.base.backoff + name: argocd-cmd-params-cm + optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_SERVER_SIDE_DIFF + valueFrom: + configMapKeyRef: + key: controller.diff.server.side + name: argocd-cmd-params-cm + optional: true + image: quay.io/argoproj/argocd:v2.10.2 + imagePullPolicy: IfNotPresent + name: application-controller + ports: + - containerPort: 8082 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: metrics + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /app/config/controller/tls + name: argocd-repo-server-tls + - mountPath: /home/argocd + name: argocd-home + workingDir: /home/argocd + dnsPolicy: ClusterFirst + serviceAccountName: argocd-application-controller + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: argocd-home + - name: argocd-repo-server-tls + secret: + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - key: ca.crt + path: ca.crt + optional: true + secretName: argocd-repo-server-tls diff --git a/hooks/application.go b/hooks/application.go new file mode 100644 index 0000000..f9848d0 --- /dev/null +++ b/hooks/application.go @@ -0,0 +1,57 @@ +package hooks + +import ( + "context" + "fmt" + "net/http" + + "github.com/cybozu-go/cattage/pkg/argocd" + "github.com/cybozu-go/cattage/pkg/config" + admissionv1 "k8s.io/api/admission/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +//+kubebuilder:webhook:path=/validate-argoproj-io-application,mutating=false,failurePolicy=fail,sideEffects=None,groups=argoproj.io,resources=applications,verbs=create;update,versions=v1alpha1,name=vapplication.kb.io,admissionReviewVersions={v1} + +type applicationValidator struct { + client client.Client + dec *admission.Decoder + config *config.Config +} + +var _ admission.Handler = &applicationValidator{} + +func (v *applicationValidator) Handle(ctx context.Context, req admission.Request) admission.Response { + if !v.config.ArgoCD.PreventAppCreationInArgoCDNamespace { + return admission.Allowed("") + } + + app := argocd.Application() + if err := v.dec.Decode(req, app); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if app.GetNamespace() == v.config.ArgoCD.Namespace { + if req.Operation != admissionv1.Create { + return admission.Allowed("").WithWarnings(fmt.Sprintf("creating Application in %s namespace is forbidden", v.config.ArgoCD.Namespace)) + } + return admission.Denied(fmt.Sprintf("cannot create Application in %s namespace", v.config.ArgoCD.Namespace)) + } + + return admission.Allowed("") +} + +// SetupApplicationWebhook registers the webhooks for Application +func SetupApplicationWebhook(mgr manager.Manager, dec *admission.Decoder, config *config.Config) { + serv := mgr.GetWebhookServer() + + v := &applicationValidator{ + client: mgr.GetClient(), + dec: dec, + config: config, + } + serv.Register("/validate-argoproj-io-application", &webhook.Admission{Handler: v}) +} diff --git a/hooks/application_test.go b/hooks/application_test.go new file mode 100644 index 0000000..93b5359 --- /dev/null +++ b/hooks/application_test.go @@ -0,0 +1,54 @@ +package hooks + +import ( + "context" + + "github.com/cybozu-go/cattage/pkg/argocd" + "github.com/cybozu-go/cattage/pkg/constants" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func fillApplication(name, namespace, project string) (*unstructured.Unstructured, error) { + app := argocd.Application() + app.SetName(name) + app.SetNamespace(namespace) + err := unstructured.SetNestedField(app.UnstructuredContent(), project, "spec", "project") + if err != nil { + return nil, err + } + err = unstructured.SetNestedField(app.UnstructuredContent(), "https://github.com/neco-test/apps-sandbox.git", "spec", "source", "repoURL") + if err != nil { + return nil, err + } + err = unstructured.SetNestedMap(app.UnstructuredContent(), map[string]interface{}{}, "spec", "destination") + if err != nil { + return nil, err + } + return app, nil +} + +var _ = Describe("Application webhook", func() { + ctx := context.Background() + + It("should allow creating an application in any namespace", func() { + app, err := fillApplication("tenant", "sub-1", "team-a") + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Create(ctx, app) + Expect(err).NotTo(HaveOccurred()) + + Expect(controllerutil.ContainsFinalizer(app, constants.Finalizer)).To(BeFalse()) + }) + + It("should deny creating an application in argocd namespace", func() { + app, err := fillApplication("default-app", "argocd", "default") + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Create(ctx, app) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("cannot create Application in argocd namespace")) + }) +}) diff --git a/hooks/suite_test.go b/hooks/suite_test.go index 8782e2f..55a3346 100644 --- a/hooks/suite_test.go +++ b/hooks/suite_test.go @@ -99,10 +99,12 @@ var _ = BeforeSuite(func() { config := &config.Config{ Namespace: config.NamespaceConfig{}, ArgoCD: config.ArgoCDConfig{ - Namespace: "argocd", + Namespace: "argocd", + PreventAppCreationInArgoCDNamespace: true, }, } SetupTenantWebhook(mgr, admission.NewDecoder(scheme), config) + SetupApplicationWebhook(mgr, admission.NewDecoder(scheme), config) //+kubebuilder:scaffold:webhook diff --git a/pkg/config/types.go b/pkg/config/types.go index 6a93d83..cb47e75 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -37,6 +37,9 @@ type ArgoCDConfig struct { // AppProjectTemplate is a template for AppProject resources that is created for each tenant AppProjectTemplate string `json:"appProjectTemplate"` + + // PreventAppCreationInArgoCDNamespace is a flag to prevent creating applications in the Argo CD namespace + PreventAppCreationInArgoCDNamespace bool `json:"preventAppCreationInArgoCDNamespace"` } // Validate validates the configurations. diff --git a/pkg/constants/indexer.go b/pkg/constants/indexer.go index dc4e407..06eff6b 100644 --- a/pkg/constants/indexer.go +++ b/pkg/constants/indexer.go @@ -1,4 +1,5 @@ package constants -const RootNamespaces = "cattage.namespaces.root" -const TenantNamespaces = "cattage.namespaces.tenant" +const RootNamespaceIndex = "cattage.namespaces.root" +const TenantNamespaceIndex = "cattage.namespaces.tenant" +const ControllerNameIndex = "cattage.tenants.controller" diff --git a/pkg/constants/meta.go b/pkg/constants/meta.go index 35859a1..d12e86a 100644 --- a/pkg/constants/meta.go +++ b/pkg/constants/meta.go @@ -11,3 +11,9 @@ const OwnerTenant = MetaPrefix + "tenant" const OwnerAppNamespace = MetaPrefix + "owner-namespace" const TenantFieldManager = MetaPrefix + "tenant-controller" + +const DefaultApplicationControllerName = "default" + +const ManagedByLabel = "app.kubernetes.io/managed-by" +const PartOfLabel = "app.kubernetes.io/part-of" +const ControllerNameLabel = MetaPrefix + "controller-name"