From b25d2a55478caf66544c645ee0bee2c4133c854b Mon Sep 17 00:00:00 2001 From: bells17 Date: Tue, 7 Sep 2021 02:12:51 +0900 Subject: [PATCH 1/6] Add ResourceQuota propagation to the default setting --- config.yaml | 2 ++ config/rbac/role.yaml | 12 ++++++++ controllers/subnamespace_controller.go | 1 + docs/config.md | 39 +------------------------- docs/subnamespaces.md | 2 -- e2e/config.yaml | 2 ++ 6 files changed, 18 insertions(+), 40 deletions(-) diff --git a/config.yaml b/config.yaml index 0a876ab..ce83fca 100644 --- a/config.yaml +++ b/config.yaml @@ -23,3 +23,5 @@ watches: kind: RoleBinding - version: v1 kind: Secret +- version: v1 + kind: ResourceQuota diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a90ecac..34c456c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -18,6 +18,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - accurate.cybozu.com resources: diff --git a/controllers/subnamespace_controller.go b/controllers/subnamespace_controller.go index c751a92..b5f90b1 100644 --- a/controllers/subnamespace_controller.go +++ b/controllers/subnamespace_controller.go @@ -29,6 +29,7 @@ type SubNamespaceReconciler struct { //+kubebuilder:rbac:groups=accurate.cybozu.com,resources=subnamespaces/status,verbs=get;update;patch //+kubebuilder:rbac:groups=accurate.cybozu.com,resources=subnamespaces/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=resourcequotas,verbs=get;list;watch;create;update;patch;delete // Reconcile implements reconcile.Reconciler interface. func (r *SubNamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/docs/config.md b/docs/config.md index 6b12fb1..a14c1e2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -21,43 +21,6 @@ The file will be embedded in ConfigMap laster using `kustomize`. `config/rbac/role_binding.yaml` contains ClusterRoleBindings for `accurate-controller`. -A built-in ClusterRole `admin` is bound by default to allow `accurate-controller` to watch and propagate namespace-scope resources. However, `admin` does not contain verbs for [ResourceQuota][] and may not contain custom resources. +A built-in ClusterRole `admin` is bound by default to allow `accurate-controller` to watch and propagate namespace-scope resources. If you need to watch and propagate resources not included in `admin` ClusterRole, add additional ClusterRole/ClusterRoleBinding to `accurate-controller-manager` ServiceAccount in `accurate` namespace. - -The following example is to watch and propagate ResourceQuotas. - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: accurate-additional-resources -rules: -- apiGroups: - - "" - resources: - - resourcequotas - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: accurate-additional-resources -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: accurate-additional-resources -subjects: -- kind: ServiceAccount - name: accurate-controller-manager - namespace: accurate -``` - -[ResourceQuota]: https://kubernetes.io/docs/concepts/policy/resource-quotas/ diff --git a/docs/subnamespaces.md b/docs/subnamespaces.md index 052b7e1..bf16637 100644 --- a/docs/subnamespaces.md +++ b/docs/subnamespaces.md @@ -53,8 +53,6 @@ subjects: apiGroup: rbac.authorization.k8s.io ``` -You may want to prepare more objects such as ResourceQuotas. - ## Reverting a root Namespace to a normal one Using `kubectl accurate`: diff --git a/e2e/config.yaml b/e2e/config.yaml index 009b879..ff86938 100644 --- a/e2e/config.yaml +++ b/e2e/config.yaml @@ -16,3 +16,5 @@ watches: kind: RoleBinding - version: v1 kind: Secret +- version: v1 + kind: ResourceQuota From fc21c16bd80109c7c7fcc4649dd4e3c5883f23f0 Mon Sep 17 00:00:00 2001 From: bells17 Date: Fri, 24 Sep 2021 15:53:01 +0900 Subject: [PATCH 2/6] Update helm version --- charts/accurate/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/accurate/Chart.yaml b/charts/accurate/Chart.yaml index f726041..adab87b 100644 --- a/charts/accurate/Chart.yaml +++ b/charts/accurate/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to From 3a38d21b624b12c4e9d48f813805282516d025c4 Mon Sep 17 00:00:00 2001 From: bells17 Date: Fri, 24 Sep 2021 15:57:12 +0900 Subject: [PATCH 3/6] Regenerate manifests --- charts/accurate/templates/generated/generated.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/charts/accurate/templates/generated/generated.yaml b/charts/accurate/templates/generated/generated.yaml index 622105f..18240d6 100644 --- a/charts/accurate/templates/generated/generated.yaml +++ b/charts/accurate/templates/generated/generated.yaml @@ -74,6 +74,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - accurate.cybozu.com resources: From b9ca8d9d2c28ecce57d8f5405bf2f7294f11ee2a Mon Sep 17 00:00:00 2001 From: bells17 Date: Tue, 5 Oct 2021 18:14:34 +0900 Subject: [PATCH 4/6] Fix --- .../templates/generated/generated.yaml | 12 -------- charts/accurate/values.yaml | 26 ++++++++-------- config/rbac/role.yaml | 12 -------- controllers/subnamespace_controller.go | 1 - docs/config.md | 30 ++++++++++++++++++- docs/subnamespaces.md | 2 ++ e2e/config.yaml | 20 ------------- 7 files changed, 44 insertions(+), 59 deletions(-) delete mode 100644 e2e/config.yaml diff --git a/charts/accurate/templates/generated/generated.yaml b/charts/accurate/templates/generated/generated.yaml index 18240d6..622105f 100644 --- a/charts/accurate/templates/generated/generated.yaml +++ b/charts/accurate/templates/generated/generated.yaml @@ -74,18 +74,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - resourcequotas - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - accurate.cybozu.com resources: diff --git a/charts/accurate/values.yaml b/charts/accurate/values.yaml index 5972cf2..8805b53 100644 --- a/charts/accurate/values.yaml +++ b/charts/accurate/values.yaml @@ -56,16 +56,16 @@ controller: # controller.additionalRBAC.rules -- Specify the RBAC rules to be added to the controller. # ClusterRole and ClusterRoleBinding are created with the names `{{ release name }}-additional-resources`. # The rules defined here will be used for the ClusterRole rules. - rules: [] - # - apiGroups: - # - "" - # resources: - # - resourcequotas - # verbs: - # - get - # - list - # - watch - # - create - # - update - # - patch - # - delete + rules: + - apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch + - create + - update + - patch + - delete diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 34c456c..a90ecac 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -18,18 +18,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - resourcequotas - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - accurate.cybozu.com resources: diff --git a/controllers/subnamespace_controller.go b/controllers/subnamespace_controller.go index b5f90b1..c751a92 100644 --- a/controllers/subnamespace_controller.go +++ b/controllers/subnamespace_controller.go @@ -29,7 +29,6 @@ type SubNamespaceReconciler struct { //+kubebuilder:rbac:groups=accurate.cybozu.com,resources=subnamespaces/status,verbs=get;update;patch //+kubebuilder:rbac:groups=accurate.cybozu.com,resources=subnamespaces/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="",resources=resourcequotas,verbs=get;list;watch;create;update;patch;delete // Reconcile implements reconcile.Reconciler interface. func (r *SubNamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/docs/config.md b/docs/config.md index 85e3be7..aef0c19 100644 --- a/docs/config.md +++ b/docs/config.md @@ -54,6 +54,34 @@ controller: ## ClusterRoleBindings -A built-in ClusterRole `admin` is bound by default to allow `accurate-controller` to watch and propagate namespace-scope resources. +A built-in ClusterRole `admin` is bound by default to allow `accurate-controller` to watch and propagate namespace-scope resources. However, `admin` does not contain verbs for [ResourceQuota][] and may not contain custom resources. If you need to watch and propagate resources not included in `admin` ClusterRole, add additional ClusterRole/ClusterRoleBinding to `accurate-controller-manager` ServiceAccount. +Set the `controller.additionalRBAC.rules` in the Helm Chart values. + +The following example Helm chart values is to watch and propagate ResourceQuotas. + +```yaml + +controller: + additionalRBAC: + # controller.additionalRBAC.rules -- Specify the RBAC rules to be added to the controller. + # ClusterRole and ClusterRoleBinding are created with the names `{{ release name }}-additional-resources`. + # The rules defined here will be used for the ClusterRole rules. + rules: + - apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + +``` + +[ResourceQuota]: https://kubernetes.io/docs/concepts/policy/resource-quotas/ diff --git a/docs/subnamespaces.md b/docs/subnamespaces.md index bf16637..052b7e1 100644 --- a/docs/subnamespaces.md +++ b/docs/subnamespaces.md @@ -53,6 +53,8 @@ subjects: apiGroup: rbac.authorization.k8s.io ``` +You may want to prepare more objects such as ResourceQuotas. + ## Reverting a root Namespace to a normal one Using `kubectl accurate`: diff --git a/e2e/config.yaml b/e2e/config.yaml deleted file mode 100644 index ff86938..0000000 --- a/e2e/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Labels to be propagated to sub-namespaces. -labelKeys: -- team - -annotationKeys: -- test - -# List of GVK for namespace-scoped resources that can be propagated. -# Any namespace-scoped resource is allowed. -watches: -- group: rbac.authorization.k8s.io - version: v1 - kind: Role -- group: rbac.authorization.k8s.io - version: v1 - kind: RoleBinding -- version: v1 - kind: Secret -- version: v1 - kind: ResourceQuota From b93e65409925bdc122b01c226a9c2356de29eb9a Mon Sep 17 00:00:00 2001 From: bells17 Date: Mon, 11 Oct 2021 19:37:44 +0900 Subject: [PATCH 5/6] Fix values.yaml and add tests --- charts/accurate/values.yaml | 2 ++ e2e/e2e_test.go | 23 +++++++++++++++++++++++ e2e/testdata/resourceQuota.yaml | 11 +++++++++++ e2e/values.yaml | 14 ++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 e2e/testdata/resourceQuota.yaml diff --git a/charts/accurate/values.yaml b/charts/accurate/values.yaml index 8805b53..f27ade9 100644 --- a/charts/accurate/values.yaml +++ b/charts/accurate/values.yaml @@ -51,6 +51,8 @@ controller: kind: RoleBinding - version: v1 kind: Secret + - version: v1 + kind: ResourceQuota additionalRBAC: # controller.additionalRBAC.rules -- Specify the RBAC rules to be added to the controller. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index efff9c7..670d5ac 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -16,6 +16,9 @@ import ( //go:embed testdata/role.yaml var roleYAML []byte +//go:embed testdata/resourceQuota.yaml +var resourceQuota []byte + var sealedJSON []byte func init() { @@ -70,6 +73,7 @@ var _ = Describe("kubectl accurate", func() { kubectlSafe(roleYAML, "apply", "-f", "-") kubectlSafe(nil, "create", "-n", "tmpl3", "secret", "generic", "s1", "--from-literal=foo=bar") + kubectlSafe(resourceQuota, "apply", "-f", "-") By("setting up templates") kubectlSafe(nil, "accurate", "template", "set", "tmpl3", "tmpl2") @@ -77,6 +81,7 @@ var _ = Describe("kubectl accurate", func() { By("checking propagation from templates") kubectlSafe(nil, "annotate", "-n", "tmpl3", "secret", "s1", "accurate.cybozu.com/propagate=update") + kubectlSafe(nil, "annotate", "-n", "tmpl3", "quota", "rq1", "accurate.cybozu.com/propagate=update") Eventually(func() error { _, err := kubectl(nil, "get", "-n", "root2", "roles", "role1") @@ -86,6 +91,10 @@ var _ = Describe("kubectl accurate", func() { _, err := kubectl(nil, "get", "-n", "root2", "secrets", "s1") return err }).Should(Succeed()) + Eventually(func() error { + _, err := kubectl(nil, "get", "-n", "root2", "quota", "rq1") + return err + }).Should(Succeed()) Eventually(func() string { out, err := kubectl(nil, "get", "ns", "root2", "-o", "json") if err != nil { @@ -115,6 +124,20 @@ var _ = Describe("kubectl accurate", func() { return errors.New("s1 exists") } } + + out, err = kubectl(nil, "get", "-n", "root2", "quota", "-o", "json") + if err != nil { + return err + } + rql := &corev1.ResourceQuotaList{} + if err := json.Unmarshal(out, rql); err != nil { + return err + } + for _, rq := range rql.Items { + if rq.Name == "rq1" { + return errors.New("rq1 exists") + } + } return nil }).Should(Succeed()) kubectlSafe(nil, "get", "-n", "root2", "roles", "role1") diff --git a/e2e/testdata/resourceQuota.yaml b/e2e/testdata/resourceQuota.yaml new file mode 100644 index 0000000..6b3f6d6 --- /dev/null +++ b/e2e/testdata/resourceQuota.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + namespace: tmpl3 + name: rq1 +spec: + hard: + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi diff --git a/e2e/values.yaml b/e2e/values.yaml index 67dae91..6806ff9 100644 --- a/e2e/values.yaml +++ b/e2e/values.yaml @@ -23,6 +23,8 @@ controller: kind: RoleBinding - version: v1 kind: Secret + - version: v1 + kind: ResourceQuota additionalRBAC: rules: @@ -34,3 +36,15 @@ controller: - get - list - watch + - apiGroups: + - "" + resources: + - resourcequotas + verbs: + - get + - list + - watch + - create + - update + - patch + - delete From d20488412c739c60cac8963b3c2a5302fd388838 Mon Sep 17 00:00:00 2001 From: bells17 Date: Wed, 13 Oct 2021 01:05:08 +0900 Subject: [PATCH 6/6] Update chart version --- charts/accurate/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/accurate/Chart.yaml b/charts/accurate/Chart.yaml index adab87b..1f1b6e4 100644 --- a/charts/accurate/Chart.yaml +++ b/charts/accurate/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.1 +version: 0.1.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to