Skip to content
This repository was archived by the owner on Feb 2, 2025. It is now read-only.

Commit 42023be

Browse files
karan56625karan56625
committed
add support to run fluentbit as daemonset
Signed-off-by: karan k <karan.k@oracle.com>
1 parent 1630d7b commit 42023be

File tree

16 files changed

+7195
-15
lines changed

16 files changed

+7195
-15
lines changed

‎apis/fluentbit/v1alpha2/zz_generated.deepcopy.go

Lines changed: 25 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎apis/fluentd/v1alpha1/fluentd_types.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ type FluentdSpec struct {
4343
// By default will build the related service according to the globalinputs definition.
4444
DisableService bool `json:"disableService,omitempty"`
4545
// Numbers of the Fluentd instance
46+
// Applicable when the mode is "collector", and will be ignored when the mode is "agent"
4647
Replicas *int32 `json:"replicas,omitempty"`
4748
// Numbers of the workers in Fluentd instance
4849
Workers *int32 `json:"workers,omitempty"`
@@ -92,13 +93,23 @@ type FluentdSpec struct {
9293
// claims in a way that maintains the identity of a pod. Every claim in
9394
// this list must have at least one matching (by name) volumeMount in one
9495
// container in the template.
96+
// Applicable when the mode is "collector", and will be ignored when the mode is "agent"
9597
VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
9698
// Service represents configurations on the fluentd service.
9799
Service FluentDService `json:"service,omitempty"`
98100
// PodSecurityContext represents the security context for the fluentd pods.
99101
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
100102
// SchedulerName represents the desired scheduler for fluentd pods.
101103
SchedulerName string `json:"schedulerName,omitempty"`
104+
// Mode to determine whether to run Fluentd as collector or agent.
105+
// +kubebuilder:validation:Enum:=collector;agent
106+
// +kubebuilder:default:=collector
107+
Mode string `json:"mode,omitempty"`
108+
// ContainerSecurityContext represents the security context for the fluentd container.
109+
ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"`
110+
// Storage for position db. You will use it if tail input is enabled.
111+
// Applicable when the mode is "agent", and will be ignored when the mode is "collector"
112+
PositionDB corev1.VolumeSource `json:"positionDB,omitempty"`
102113
}
103114

104115
// FluentDService the service of the FluentD

‎apis/fluentd/v1alpha1/zz_generated.deepcopy.go

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml

Lines changed: 1715 additions & 1 deletion
Large diffs are not rendered by default.

‎charts/fluent-operator/templates/fluentd-fluentd.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ spec:
1313
port: {{ .Values.fluentd.port }}
1414
replicas: {{ .Values.fluentd.replicas }}
1515
image: {{ .Values.fluentd.image.repository }}:{{ .Values.fluentd.image.tag }}
16+
mode: {{ .Values.fluentd.mode }}
1617
resources:
1718
{{- toYaml .Values.fluentd.resources | nindent 4 }}
1819
fluentdCfgSelector:

‎charts/fluent-operator/values.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,10 +268,16 @@ fluentd:
268268
crdsEnable: true
269269
enable: false
270270
name: fluentd
271+
# Valid modes include "collector" and "agent".
272+
# The "collector" mode will deploy Fluentd as a StatefulSet as before.
273+
# The new "agent" mode will deploy Fluentd as a DaemonSet.
274+
mode: "collector"
271275
port: 24224
272276
image:
273277
repository: "kubesphere/fluentd"
274278
tag: "v1.15.3"
279+
# Numbers of the Fluentd instance
280+
# Applicable when the mode is "collector", and will be ignored when the mode is "agent"
275281
replicas: 1
276282
forward:
277283
port: 24224

‎config/crd/bases/fluentd.fluent.io_fluentds.yaml

Lines changed: 1715 additions & 1 deletion
Large diffs are not rendered by default.

‎controllers/consts.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@ var (
1111
fluentdOwnerKey = ".fluentd.metadata.controller"
1212
fluentbitApiGVStr = fluentbitv1alpha2.SchemeGroupVersion.String()
1313
fluentdApiGVStr = fluentdv1alpha1.SchemeGroupVersion.String()
14+
fluentdAgentMode = "agent"
1415
)

‎controllers/fluent_controller_finalizer.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,16 @@ func (r *FluentdReconciler) delete(ctx context.Context, fd *fluentdv1alpha1.Flue
8787
return err
8888
}
8989

90+
ds := appsv1.DaemonSet{
91+
ObjectMeta: metav1.ObjectMeta{
92+
Name: fd.Name,
93+
Namespace: fd.Namespace,
94+
},
95+
}
96+
if err := r.Delete(ctx, &ds); err != nil && !errors.IsNotFound(err) {
97+
return err
98+
}
99+
90100
svc := corev1.Service{
91101
ObjectMeta: metav1.ObjectMeta{
92102
Name: fd.Name,
@@ -129,8 +139,18 @@ func (r *FluentdReconciler) mutate(obj client.Object, fd *fluentdv1alpha1.Fluent
129139
return nil
130140
}
131141
case *appsv1.StatefulSet:
132-
expected := operator.MakeStatefulset(*fd)
142+
expected := operator.MakeStatefulSet(*fd)
133143

144+
return func() error {
145+
o.Labels = expected.Labels
146+
o.Spec = expected.Spec
147+
if err := ctrl.SetControllerReference(fd, o, r.Scheme); err != nil {
148+
return err
149+
}
150+
return nil
151+
}
152+
case *appsv1.DaemonSet:
153+
expected := operator.MakeFluentdDaemonSet(*fd)
134154
return func() error {
135155
o.Labels = expected.Labels
136156
o.Spec = expected.Spec

‎controllers/fluentd_controller.go

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -105,17 +105,44 @@ func (r *FluentdReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
105105
return ctrl.Result{}, err
106106
}
107107

108-
// Deploy Fluentd Statefulset
109-
sts := operator.MakeStatefulset(fd)
110-
if _, err := controllerutil.CreateOrPatch(ctx, r.Client, sts, r.mutate(sts, &fd)); err != nil {
111-
return ctrl.Result{}, err
108+
var err error
109+
if fd.Spec.Mode == "agent" {
110+
// Deploy Fluentd DaemonSet
111+
ds := operator.MakeFluentdDaemonSet(fd)
112+
_, err = controllerutil.CreateOrPatch(ctx, r.Client, ds, r.mutate(ds, &fd))
113+
if err != nil {
114+
return ctrl.Result{}, err
115+
}
116+
sts := appsv1.StatefulSet{
117+
ObjectMeta: metav1.ObjectMeta{
118+
Name: fd.Name,
119+
Namespace: fd.Namespace,
120+
},
121+
}
122+
if err = r.Delete(ctx, &sts); err != nil && !errors.IsNotFound(err) {
123+
return ctrl.Result{}, err
124+
}
125+
} else {
126+
// Deploy Fluentd StatefulSet
127+
sts := operator.MakeStatefulSet(fd)
128+
_, err = controllerutil.CreateOrPatch(ctx, r.Client, sts, r.mutate(sts, &fd))
129+
ds := appsv1.DaemonSet{
130+
ObjectMeta: metav1.ObjectMeta{
131+
Name: fd.Name,
132+
Namespace: fd.Namespace,
133+
},
134+
}
135+
if err = r.Delete(ctx, &ds); err != nil && !errors.IsNotFound(err) {
136+
return ctrl.Result{}, err
137+
}
112138
}
113-
114139
// Deploy Fluentd Service
115140
if !fd.Spec.DisableService {
116141
svc := operator.MakeFluentdService(fd)
117-
if _, err := controllerutil.CreateOrPatch(ctx, r.Client, svc, r.mutate(svc, &fd)); err != nil {
118-
return ctrl.Result{}, err
142+
if len(svc.Spec.Ports) > 0 {
143+
if _, err = controllerutil.CreateOrPatch(ctx, r.Client, svc, r.mutate(svc, &fd)); err != nil {
144+
return ctrl.Result{}, err
145+
}
119146
}
120147
}
121148

@@ -139,6 +166,22 @@ func (r *FluentdReconciler) SetupWithManager(mgr ctrl.Manager) error {
139166
return err
140167
}
141168

169+
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.DaemonSet{}, fluentdOwnerKey, func(rawObj client.Object) []string {
170+
// grab the job object, extract the owner.
171+
ds := rawObj.(*appsv1.DaemonSet)
172+
owner := metav1.GetControllerOf(ds)
173+
if owner == nil {
174+
return nil
175+
}
176+
177+
if owner.APIVersion != fluentdApiGVStr || owner.Kind != "Fluentd" {
178+
return nil
179+
}
180+
return []string{owner.Name}
181+
}); err != nil {
182+
return err
183+
}
184+
142185
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appsv1.StatefulSet{}, fluentdOwnerKey, func(rawObj client.Object) []string {
143186
// grab the job object, extract the owner.
144187
sts := rawObj.(*appsv1.StatefulSet)
@@ -174,6 +217,7 @@ func (r *FluentdReconciler) SetupWithManager(mgr ctrl.Manager) error {
174217
return ctrl.NewControllerManagedBy(mgr).
175218
For(&fluentdv1alpha1.Fluentd{}).
176219
Owns(&corev1.ServiceAccount{}).
220+
Owns(&appsv1.DaemonSet{}).
177221
Owns(&appsv1.StatefulSet{}).
178222
Owns(&corev1.Service{}).
179223
Complete(r)

‎go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,14 @@ require (
5858
go.uber.org/atomic v1.9.0 // indirect
5959
go.uber.org/multierr v1.7.0 // indirect
6060
go.uber.org/zap v1.24.0 // indirect
61+
golang.org/x/mod v0.10.0 // indirect
6162
golang.org/x/net v0.10.0 // indirect
6263
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
6364
golang.org/x/sys v0.8.0 // indirect
6465
golang.org/x/term v0.8.0 // indirect
6566
golang.org/x/text v0.9.0 // indirect
6667
golang.org/x/time v0.3.0 // indirect
68+
golang.org/x/tools v0.9.1 // indirect
6769
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
6870
google.golang.org/appengine v1.6.7 // indirect
6971
google.golang.org/protobuf v1.28.1 // indirect
@@ -72,7 +74,9 @@ require (
7274
gopkg.in/yaml.v2 v2.4.0 // indirect
7375
gopkg.in/yaml.v3 v3.0.1 // indirect
7476
k8s.io/apiextensions-apiserver v0.26.1 // indirect
77+
k8s.io/code-generator v0.26.1 // indirect
7578
k8s.io/component-base v0.26.1 // indirect
79+
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
7680
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
7781
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
7882
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

‎go.sum

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
8989
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
9090
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
9191
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
92+
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
9293
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
9394
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
9495
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
@@ -349,6 +350,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
349350
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
350351
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
351352
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
353+
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
354+
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
352355
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
353356
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
354357
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -503,6 +506,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
503506
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
504507
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
505508
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
509+
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
506510
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
507511
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
508512
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -513,6 +517,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
513517
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
514518
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
515519
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
520+
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
516521
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
517522
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
518523
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -640,8 +645,13 @@ k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
640645
k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
641646
k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s=
642647
k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ=
648+
k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo=
649+
k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I=
643650
k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4=
644651
k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU=
652+
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08=
653+
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
654+
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
645655
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
646656
k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
647657
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=
@@ -657,5 +667,6 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm
657667
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
658668
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
659669
sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
670+
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
660671
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
661672
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

0 commit comments

Comments
 (0)