Skip to content

Commit 6454ded

Browse files
bastjanbastjan
authored
Implement legacy ResourceQuota and LimitRange generation (#126)
Replaces https://hub.syn.tools/appuio-cloud/references/policies/11_generate_quota_limit_range_in_ns.html. Also includes a webhook to deny edits to the synced resources.
1 parent 597c554 commit 6454ded

12 files changed

+613
-3
lines changed

‎Makefile.vars.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ CONTAINER_IMG ?= local.dev/$(PROJECT_OWNER)/$(PROJECT_NAME):$(IMG_TAG)
1616

1717
LOCALBIN ?= $(shell pwd)/bin
1818
ENVTEST ?= $(LOCALBIN)/setup-envtest
19-
ENVTEST_K8S_VERSION = 1.26.1
19+
ENVTEST_K8S_VERSION = 1.28.3
2020

2121
## KIND:setup
2222

‎config.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/appuio/appuio-cloud-agent/limits"
88
"go.uber.org/multierr"
99
"gopkg.in/inf.v0"
10+
corev1 "k8s.io/api/core/v1"
1011
"k8s.io/apimachinery/pkg/api/resource"
1112
"sigs.k8s.io/yaml"
1213
)
@@ -71,6 +72,18 @@ type Config struct {
7172
PodRunOnceActiveDeadlineSecondsOverrideAnnotation string
7273
// PodRunOnceActiveDeadlineSecondsDefault is the default activeDeadlineSeconds for RunOnce pods.
7374
PodRunOnceActiveDeadlineSecondsDefault int
75+
76+
// LegacyResourceQuotaAnnotationBase is the base label for the default resource quotas.
77+
// The actual annotation is `$base/$quotaname.$resource`.
78+
LegacyResourceQuotaAnnotationBase string
79+
// LegacyDefaultResourceQuotas is a map containing the default resource quotas for each organization.
80+
// The keys are the name of the manifest and the values are the resource quotas spec.
81+
LegacyDefaultResourceQuotas map[string]corev1.ResourceQuotaSpec
82+
83+
// LegacyLimitRangeName is the name of the default limit range.
84+
LegacyLimitRangeName string
85+
// LegacyDefaultLimitRange is the default limit range.
86+
LegacyDefaultLimitRange corev1.LimitRangeSpec
7487
}
7588

7689
func ConfigFromFile(path string) (c Config, warn []string, err error) {

‎config.yaml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,70 @@ AllowedLabels: [appuio.io/organization]
5252
PodRunOnceActiveDeadlineSecondsOverrideAnnotation: appuio.io/active-deadline-seconds-override
5353
# PodRunOnceActiveDeadlineSecondsDefault is the default activeDeadlineSeconds for RunOnce pods.
5454
PodRunOnceActiveDeadlineSecondsDefault: 1800
55+
56+
# LegacyResourceQuotaAnnotationBase is the base label for the default resource quotas.
57+
# The actual annotation is `$base/$quotaname.$resource`.
58+
LegacyResourceQuotaAnnotationBase: resourcequota.appuio.io
59+
# LegacyDefaultResourceQuotas is a map containing the default resource quotas for each organization.
60+
# The keys are the name of the manifest and the values are the resource quotas spec.
61+
LegacyDefaultResourceQuotas:
62+
# See https://kb.vshn.ch/appuio-cloud/references/quality-requirements/performance/resource-quota.html
63+
organization-objects:
64+
hard:
65+
count/configmaps: "150"
66+
count/jobs.batch: "150"
67+
count/secrets: "150"
68+
count/services: "20"
69+
count/services.loadbalancers: "0"
70+
count/services.nodeports: "0"
71+
count/replicationcontrollers: "100"
72+
openshift.io/imagestreams: "20"
73+
openshift.io/imagestreamtags: "50"
74+
75+
requests.storage: 1000Gi
76+
persistentvolumeclaims: "10"
77+
localblock-storage.storageclass.storage.k8s.io/persistentvolumeclaims: "0"
78+
requests.ephemeral-storage: "250Mi"
79+
limits.ephemeral-storage: "500Mi"
80+
81+
# Limit the total amount of Rook-Ceph backed storage which can be
82+
# requested per namespace
83+
cephfs-fspool-cluster.storageclass.storage.k8s.io/requests.storage: 25Gi
84+
rbd-storagepool-cluster.storageclass.storage.k8s.io/requests.storage: 25Gi
85+
86+
organization-compute:
87+
hard:
88+
requests.cpu: 4
89+
requests.memory: 4Gi
90+
limits.cpu: 8
91+
limits.memory: 20Gi
92+
pods: "45"
93+
scopes:
94+
- NotTerminating
95+
96+
organization-compute-terminating:
97+
hard:
98+
limits.cpu: 4000m
99+
limits.memory: 4Gi
100+
pods: "5"
101+
requests.cpu: 500m
102+
requests.memory: 2Gi
103+
scopes:
104+
- Terminating
105+
106+
# LegacyLimitRangeName is the name of the default limit range.
107+
LegacyLimitRangeName: organization
108+
# LegacyDefaultLimitRange is the default limit range.
109+
LegacyDefaultLimitRange:
110+
limits:
111+
- type: Container
112+
min:
113+
cpu: "10m"
114+
memory: "4Mi"
115+
ephemeral-storage: "100Ki"
116+
default:
117+
cpu: "600m"
118+
memory: "768Mi"
119+
defaultRequest:
120+
cpu: "10m"
121+
memory: "100Mi"

‎config/webhook/manifests.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,29 @@ kind: ValidatingWebhookConfiguration
9494
metadata:
9595
name: validating-webhook-configuration
9696
webhooks:
97+
- admissionReviewVersions:
98+
- v1
99+
clientConfig:
100+
service:
101+
name: webhook-service
102+
namespace: system
103+
path: /validate-reserved-resourcequota-limitrange
104+
failurePolicy: Fail
105+
matchPolicy: Equivalent
106+
name: reserved-resourcequota-limitrange-validator.appuio.io
107+
rules:
108+
- apiGroups:
109+
- ""
110+
apiVersions:
111+
- v1
112+
operations:
113+
- CREATE
114+
- UPDATE
115+
- DELETE
116+
resources:
117+
- resourcequotas
118+
- limitranges
119+
sideEffects: None
97120
- admissionReviewVersions:
98121
- v1
99122
clientConfig:

‎controllers/legacy_resource_quota_controller.go

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"strings"
8+
9+
"go.uber.org/multierr"
10+
corev1 "k8s.io/api/core/v1"
11+
"k8s.io/apimachinery/pkg/api/resource"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
"k8s.io/client-go/tools/record"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/builder"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
18+
"sigs.k8s.io/controller-runtime/pkg/log"
19+
)
20+
21+
// LegacyResourceQuotaReconciler reconciles namespaces and synchronizes their resource quotas
22+
type LegacyResourceQuotaReconciler struct {
23+
client.Client
24+
Scheme *runtime.Scheme
25+
Recorder record.EventRecorder
26+
27+
OrganizationLabel string
28+
29+
ResourceQuotaAnnotationBase string
30+
DefaultResourceQuotas map[string]corev1.ResourceQuotaSpec
31+
32+
LimitRangeName string
33+
DefaultLimitRange corev1.LimitRangeSpec
34+
}
35+
36+
func (r *LegacyResourceQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
37+
l := log.FromContext(ctx)
38+
l.Info("Reconciling Namespace")
39+
40+
var ns corev1.Namespace
41+
if err := r.Get(ctx, req.NamespacedName, &ns); err != nil {
42+
return ctrl.Result{}, client.IgnoreNotFound(err)
43+
}
44+
if ns.DeletionTimestamp != nil {
45+
l.Info("Namespace is being deleted, skipping reconciliation")
46+
return ctrl.Result{}, nil
47+
}
48+
49+
if _, ok := ns.Labels[r.OrganizationLabel]; !ok {
50+
l.Info("Namespace does not have organization label, skipping reconciliation")
51+
return ctrl.Result{}, nil
52+
}
53+
54+
var errs []error
55+
for name, s := range r.DefaultResourceQuotas {
56+
spec := *s.DeepCopy()
57+
58+
var storageQuotas corev1.ResourceList
59+
if sqa := ns.Annotations[fmt.Sprintf("%s/%s.storageclasses", r.ResourceQuotaAnnotationBase, name)]; sqa != "" {
60+
err := json.Unmarshal([]byte(ns.Annotations[fmt.Sprintf("%s/%s.storageclasses", r.ResourceQuotaAnnotationBase, name)]), &storageQuotas)
61+
if err != nil {
62+
errs = append(errs, fmt.Errorf("failed to unmarshal storage classes: %w", err))
63+
storageQuotas = make(corev1.ResourceList)
64+
}
65+
} else {
66+
storageQuotas = make(corev1.ResourceList)
67+
}
68+
69+
rq := &corev1.ResourceQuota{
70+
ObjectMeta: ctrl.ObjectMeta{
71+
Name: name,
72+
Namespace: ns.Name,
73+
},
74+
}
75+
op, err := controllerutil.CreateOrUpdate(ctx, r.Client, rq, func() error {
76+
for k := range spec.Hard {
77+
an := fmt.Sprintf("%s/%s.%s", r.ResourceQuotaAnnotationBase, name, strings.ReplaceAll(string(k), "/", "_"))
78+
if strings.Contains(string(k), "storageclass.storage.k8s.io") {
79+
if _, ok := storageQuotas[k]; ok {
80+
spec.Hard[k] = storageQuotas[k]
81+
}
82+
} else if a := ns.Annotations[an]; a != "" {
83+
po, err := resource.ParseQuantity(a)
84+
if err != nil {
85+
errs = append(errs, fmt.Errorf("failed to parse quantity %s=%s: %w", an, a, err))
86+
continue
87+
}
88+
spec.Hard[k] = po
89+
}
90+
}
91+
92+
rq.Spec = spec
93+
return controllerutil.SetControllerReference(&ns, rq, r.Scheme)
94+
})
95+
if err != nil {
96+
errs = append(errs, fmt.Errorf("failed to reconcile ResourceQuota %s: %w", name, err))
97+
}
98+
if op != controllerutil.OperationResultNone {
99+
l.Info("Reconciled ResourceQuota", "name", name, "operation", op)
100+
}
101+
}
102+
103+
lr := &corev1.LimitRange{
104+
ObjectMeta: ctrl.ObjectMeta{
105+
Name: r.LimitRangeName,
106+
Namespace: ns.Name,
107+
},
108+
}
109+
op, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error {
110+
lr.Spec = *r.DefaultLimitRange.DeepCopy()
111+
return controllerutil.SetControllerReference(&ns, lr, r.Scheme)
112+
})
113+
if err != nil {
114+
errs = append(errs, fmt.Errorf("failed to reconcile LimitRange %s: %w", r.LimitRangeName, err))
115+
}
116+
if op != controllerutil.OperationResultNone {
117+
l.Info("Reconciled LimitRange", "name", r.LimitRangeName, "operation", op)
118+
}
119+
120+
if err := multierr.Combine(errs...); err != nil {
121+
r.Recorder.Eventf(&ns, corev1.EventTypeWarning, "ReconcileError", "Failed to reconcile ResourceQuotas and LimitRanges: %s", err.Error())
122+
return ctrl.Result{}, fmt.Errorf("failed to reconcile ResourceQuotas and LimitRanges: %w", err)
123+
}
124+
125+
return ctrl.Result{}, nil
126+
}
127+
128+
// SetupWithManager sets up the controller with the Manager.
129+
func (r *LegacyResourceQuotaReconciler) SetupWithManager(mgr ctrl.Manager) error {
130+
orgPredicate, err := labelExistsPredicate(r.OrganizationLabel)
131+
if err != nil {
132+
return fmt.Errorf("failed to create organization label predicate: %w", err)
133+
}
134+
return ctrl.NewControllerManagedBy(mgr).
135+
Named("legacyresourcequota").
136+
For(&corev1.Namespace{}, builder.WithPredicates(orgPredicate)).
137+
Owns(&corev1.ResourceQuota{}).
138+
Owns(&corev1.LimitRange{}).
139+
Complete(r)
140+
}

‎controllers/legacy_resource_quota_controller_test.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/go-logr/logr/testr"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
corev1 "k8s.io/api/core/v1"
11+
"k8s.io/apimachinery/pkg/api/resource"
12+
"k8s.io/apimachinery/pkg/types"
13+
"k8s.io/utils/ptr"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/log"
16+
)
17+
18+
func Test_LegacyResourceQuotaReconciler_Reconcile(t *testing.T) {
19+
t.Parallel()
20+
21+
subjectNamespace := newNamespace("test", map[string]string{"organization": "testorg"}, nil)
22+
23+
c, scheme, recorder := prepareClient(t, subjectNamespace)
24+
ctx := log.IntoContext(context.Background(), testr.New(t))
25+
26+
subject := LegacyResourceQuotaReconciler{
27+
Client: c,
28+
Scheme: scheme,
29+
Recorder: recorder,
30+
31+
OrganizationLabel: "organization",
32+
33+
ResourceQuotaAnnotationBase: "resourcequota.example.com",
34+
DefaultResourceQuotas: map[string]corev1.ResourceQuotaSpec{
35+
"orgq": {
36+
Hard: corev1.ResourceList{
37+
corev1.ResourceLimitsCPU: resource.MustParse("10"),
38+
corev1.ResourceRequestsMemory: resource.MustParse("10Gi"),
39+
"count/services.loadbalancers": resource.MustParse("10"),
40+
"localblock-storage.storageclass.storage.k8s.io/persistentvolumeclaims": resource.MustParse("10"),
41+
"cephfs-fspool-cluster.storageclass.storage.k8s.io/requests.storage": resource.MustParse("10"),
42+
"openshift.io/imagestreamtags": resource.MustParse("10"),
43+
},
44+
},
45+
},
46+
47+
LimitRangeName: "limitrange",
48+
DefaultLimitRange: corev1.LimitRangeSpec{
49+
Limits: []corev1.LimitRangeItem{
50+
{
51+
Type: corev1.LimitTypeContainer,
52+
Default: corev1.ResourceList{
53+
corev1.ResourceLimitsCPU: resource.MustParse("1"),
54+
},
55+
},
56+
},
57+
},
58+
}
59+
60+
_, err := subject.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: subjectNamespace.Name}})
61+
require.NoError(t, err)
62+
63+
var syncedRQ corev1.ResourceQuota
64+
require.NoError(t, c.Get(ctx, types.NamespacedName{Name: "orgq", Namespace: "test"}, &syncedRQ))
65+
require.Equal(t, subject.DefaultResourceQuotas["orgq"], syncedRQ.Spec)
66+
67+
var syncedLR corev1.LimitRange
68+
require.NoError(t, c.Get(ctx, types.NamespacedName{Name: "limitrange", Namespace: "test"}, &syncedLR))
69+
require.Equal(t, subject.DefaultLimitRange, syncedLR.Spec)
70+
71+
subjectNamespace.Annotations = map[string]string{
72+
"resourcequota.example.com/orgq.storageclasses": `{"cephfs-fspool-cluster.storageclass.storage.k8s.io/requests.storage":"5"}`,
73+
"resourcequota.example.com/orgq.limits.cpu": "5",
74+
"resourcequota.example.com/orgq.count_services.loadbalancers": "5",
75+
"resourcequota.example.com/orgq.openshift.io_imagestreamtags": "5",
76+
}
77+
require.NoError(t, c.Update(ctx, subjectNamespace))
78+
79+
_, err = subject.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: subjectNamespace.Name}})
80+
require.NoError(t, err)
81+
82+
require.NoError(t, c.Get(ctx, types.NamespacedName{Name: "orgq", Namespace: "test"}, &syncedRQ))
83+
assert.Equal(t, "5", ptr.To(syncedRQ.Spec.Hard[corev1.ResourceLimitsCPU]).String())
84+
assert.Equal(t, "5", ptr.To(syncedRQ.Spec.Hard["count/services.loadbalancers"]).String())
85+
assert.Equal(t, "5", ptr.To(syncedRQ.Spec.Hard["openshift.io/imagestreamtags"]).String())
86+
assert.Equal(t, "5", ptr.To(syncedRQ.Spec.Hard["cephfs-fspool-cluster.storageclass.storage.k8s.io/requests.storage"]).String())
87+
assert.Equal(t, "10", ptr.To(syncedRQ.Spec.Hard["localblock-storage.storageclass.storage.k8s.io/persistentvolumeclaims"]).String())
88+
assert.Equal(t, "10Gi", ptr.To(syncedRQ.Spec.Hard[corev1.ResourceRequestsMemory]).String())
89+
}

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ require (
1616
k8s.io/api v0.31.0
1717
k8s.io/apimachinery v0.31.0
1818
k8s.io/client-go v0.31.0
19+
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
1920
sigs.k8s.io/controller-runtime v0.19.0
2021
sigs.k8s.io/controller-tools v0.16.1
2122
sigs.k8s.io/kind v0.24.0
@@ -86,7 +87,6 @@ require (
8687
k8s.io/apiextensions-apiserver v0.31.0 // indirect
8788
k8s.io/klog/v2 v2.130.1 // indirect
8889
k8s.io/kube-openapi v0.0.0-20240816214639-573285566f34 // indirect
89-
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
9090
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
9191
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
9292
)

0 commit comments

Comments
 (0)