Skip to content

Commit

Permalink
basic implementation for config reloader (#276)
Browse files Browse the repository at this point in the history 
* basic implementation for config reloader #124

* Refactors config reload, fixes config-reloader
  • Loading branch information
@f41gh7
f41gh7 authored Jul 11, 2021
1 parent eebfa4f commit 8f6f133
Show file tree
Hide file tree
Showing 16 changed files with 1,017 additions and 124 deletions.
6 changes: 5 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,4 +403,8 @@ build-load-kind: build
kind load docker-image $(DOCKER_REPO):0.0.1

deploy-kind: build-load-kind
$(MAKE) deploy
$(MAKE) deploy


include internal/config-reloader/Makefile

162 changes: 90 additions & 72 deletions controllers/factory/vmagent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,15 @@ import (
"strconv"
"strings"

"github.com/hashicorp/go-version"
"gopkg.in/yaml.v2"

victoriametricsv1beta1 "github.com/VictoriaMetrics/operator/api/v1beta1"
"github.com/VictoriaMetrics/operator/controllers/factory/finalize"
"github.com/VictoriaMetrics/operator/controllers/factory/k8stools"
"github.com/VictoriaMetrics/operator/controllers/factory/psp"
"github.com/VictoriaMetrics/operator/controllers/factory/vmagent"
"github.com/VictoriaMetrics/operator/internal/config"
"github.com/go-logr/logr"
"github.com/hashicorp/go-version"
"gopkg.in/yaml.v2"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -344,24 +343,6 @@ func makeSpecForVMAgent(cr *victoriametricsv1beta1.VMAgent, c *config.BaseOperat
})
}

configReloadVolumeMounts := []corev1.VolumeMount{
{
Name: "config",
MountPath: vmAgentConfDir,
},
{
Name: "config-out",
MountPath: vmAgentConOfOutDir,
},
{
Name: "relabeling-assets",
ReadOnly: true,
MountPath: RelabelingConfigDir,
},
}

configReloadArgs := buildConfigReloaderArgs(cr, c.VMAgentDefault.ConfigReloadImage)

if cr.Spec.RelabelConfig != nil || len(cr.Spec.InlineRelabelConfig) > 0 {
args = append(args, "-remoteWrite.relabelConfig="+path.Join(RelabelingConfigDir, globalRelabelingName))
}
Expand All @@ -370,15 +351,6 @@ func makeSpecForVMAgent(cr *victoriametricsv1beta1.VMAgent, c *config.BaseOperat

specRes := buildResources(cr.Spec.Resources, config.Resource(c.VMAgentDefault.Resource), c.VMAgentDefault.UseDefaultResources)

prometheusConfigReloaderResources := corev1.ResourceRequirements{
Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}}
if c.VMAgentDefault.ConfigReloaderCPU != "0" && c.VMAgentDefault.UseDefaultResources {
prometheusConfigReloaderResources.Limits[corev1.ResourceCPU] = resource.MustParse(c.VMAgentDefault.ConfigReloaderCPU)
}
if c.VMAgentDefault.ConfigReloaderMemory != "0" && c.VMAgentDefault.UseDefaultResources {
prometheusConfigReloaderResources.Limits[corev1.ResourceMemory] = resource.MustParse(c.VMAgentDefault.ConfigReloaderMemory)
}

sort.Strings(args)

vmagentContainer := corev1.Container{
Expand All @@ -392,26 +364,13 @@ func makeSpecForVMAgent(cr *victoriametricsv1beta1.VMAgent, c *config.BaseOperat
Resources: specRes,
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
}

buildProbe(vmagentContainer, cr.Spec.EmbeddedProbes, cr.HealthPath, cr.Spec.Port, true)

configReloader := buildConfigReloaderContainer(cr, c)

operatorContainers := []corev1.Container{
{
Name: "config-reloader",
Image: c.VMAgentDefault.ConfigReloadImage,
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
Env: []corev1.EnvVar{
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
},
},
},
Command: []string{"/bin/prometheus-config-reloader"},
Args: configReloadArgs,
VolumeMounts: configReloadVolumeMounts,
Resources: prometheusConfigReloaderResources,
},
configReloader,
vmagentContainer,
}

Expand Down Expand Up @@ -482,31 +441,6 @@ func addShardSettingsToVMAgent(shardNum, shardsCount int, dep *appsv1.Deployment
}
}

func buildConfigReloaderArgs(cr *victoriametricsv1beta1.VMAgent, reloaderImage string) []string {

// by default use watched-dir
// it should simplify parsing for latest and empty version tags.
dirsArg := "watched-dir"
idx := strings.LastIndex(reloaderImage, ":")
if idx > 0 {
imageTag := reloaderImage[idx+1:]
ver, err := version.NewVersion(imageTag)
if err != nil {
log.Error(err, "cannot parse vmagent config reloader version", "reloader-image", reloaderImage)
} else if ver.LessThan(version.Must(version.NewVersion("0.43.0"))) {
dirsArg = "rules-dir"
}
}

args := []string{
fmt.Sprintf("--reload-url=%s", cr.ReloadPathWithPort(cr.Spec.Port)),
fmt.Sprintf("--config-file=%s", path.Join(vmAgentConfDir, configFilename)),
fmt.Sprintf("--config-envsubst-file=%s", path.Join(vmAgentConOfOutDir, configEnvsubstFilename)),
fmt.Sprintf("--%s=%s", dirsArg, RelabelingConfigDir),
}
return args
}

func addAdditionalObjectOwnership(cr *victoriametricsv1beta1.VMAgent, rclient client.Client, object client.Object) error {
err := rclient.Get(context.Background(), types.NamespacedName{Namespace: cr.Namespace, Name: object.GetName()}, object)
if err != nil {
Expand Down Expand Up @@ -1001,3 +935,87 @@ func BuildRemoteWrites(cr *victoriametricsv1beta1.VMAgent, rwsBasicAuth map[stri
}
return finalArgs
}

func buildConfigReloaderContainer(cr *victoriametricsv1beta1.VMAgent, c *config.BaseOperatorConf) corev1.Container {

configReloadVolumeMounts := []corev1.VolumeMount{
{
Name: "config",
MountPath: vmAgentConfDir,
},
{
Name: "config-out",
MountPath: vmAgentConOfOutDir,
},
{
Name: "relabeling-assets",
ReadOnly: true,
MountPath: RelabelingConfigDir,
},
}
configReloaderResources := corev1.ResourceRequirements{
Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}}
if c.VMAgentDefault.ConfigReloaderCPU != "0" && c.VMAgentDefault.UseDefaultResources {
configReloaderResources.Limits[corev1.ResourceCPU] = resource.MustParse(c.VMAgentDefault.ConfigReloaderCPU)
}
if c.VMAgentDefault.ConfigReloaderMemory != "0" && c.VMAgentDefault.UseDefaultResources {
configReloaderResources.Limits[corev1.ResourceMemory] = resource.MustParse(c.VMAgentDefault.ConfigReloaderMemory)
}

configReloadArgs := buildConfigReloaderArgs(cr, c)
cntr := corev1.Container{
Name: "config-reloader",
Image: c.VMAgentDefault.ConfigReloadImage,
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
Env: []corev1.EnvVar{
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
},
},
},
Command: []string{"/bin/prometheus-config-reloader"},
Args: configReloadArgs,
VolumeMounts: configReloadVolumeMounts,
Resources: configReloaderResources,
}
if c.UseCustomConfigReloader {
cntr.Image = c.CustomConfigReloaderImage
cntr.Command = []string{"/usr/local/bin/config-reloader"}
}
return cntr
}

func buildConfigReloaderArgs(cr *victoriametricsv1beta1.VMAgent, c *config.BaseOperatorConf) []string {

// by default use watched-dir
// it should simplify parsing for latest and empty version tags.
dirsArg := "watched-dir"
if !c.UseCustomConfigReloader {
reloaderImage := c.VMAgentDefault.ConfigReloadImage
idx := strings.LastIndex(reloaderImage, ":")
if idx > 0 {
imageTag := reloaderImage[idx+1:]
ver, err := version.NewVersion(imageTag)
if err != nil {
log.Error(err, "cannot parse vmagent config reloader version", "reloader-image", reloaderImage)
} else if ver.LessThan(version.Must(version.NewVersion("0.43.0"))) {
dirsArg = "rules-dir"
}
}
}

args := []string{
fmt.Sprintf("--reload-url=%s", cr.ReloadPathWithPort(cr.Spec.Port)),
fmt.Sprintf("--config-envsubst-file=%s", path.Join(vmAgentConOfOutDir, configEnvsubstFilename)),
fmt.Sprintf("--%s=%s", dirsArg, RelabelingConfigDir),
}
if c.UseCustomConfigReloader {
args = append(args, fmt.Sprintf("--config-secret-name=%s/%s", cr.Namespace, cr.PrefixedName()))
args = append(args, "--config-secret-key=vmagent.yaml.gz")
} else {
args = append(args, fmt.Sprintf("--config-file=%s", path.Join(vmAgentConfDir, configFilename)))
}
return args
}
1 change: 1 addition & 0 deletions controllers/factory/vmagent/rbac.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ func buildVMAgentClusterRole(cr *v1beta12.VMAgent) *v12.ClusterRole {
"endpointslices",
"configmaps",
"namespaces",
"secrets",
},
},
{
Expand Down
89 changes: 83 additions & 6 deletions controllers/factory/vmagent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -930,8 +930,8 @@ func TestCreateOrUpdateRelabelConfigsAssets(t *testing.T) {

func Test_buildConfigReloaderArgs(t *testing.T) {
type args struct {
cr *victoriametricsv1beta1.VMAgent
reloaderImage string
cr *victoriametricsv1beta1.VMAgent
c *config.BaseOperatorConf
}
tests := []struct {
name string
Expand All @@ -944,7 +944,45 @@ func Test_buildConfigReloaderArgs(t *testing.T) {
cr: &victoriametricsv1beta1.VMAgent{
Spec: victoriametricsv1beta1.VMAgentSpec{Port: "8429"},
},
reloaderImage: "prometheus-config-reloader:latest",
c: &config.BaseOperatorConf{
VMAgentDefault: struct {
Image string `default:"victoriametrics/vmagent"`
Version string `default:"v1.58.0"`
ConfigReloadImage string `default:"quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1"`
Port string `default:"8429"`
UseDefaultResources bool `default:"true"`
Resource struct {
Limit struct {
Mem string `default:"500Mi"`
Cpu string `default:"200m"`
}
Request struct {
Mem string `default:"200Mi"`
Cpu string `default:"50m"`
}
}
ConfigReloaderCPU string `default:"100m"`
ConfigReloaderMemory string `default:"25Mi"`
}(struct {
Image string
Version string
ConfigReloadImage string
Port string
UseDefaultResources bool
Resource struct {
Limit struct {
Mem string `default:"500Mi"`
Cpu string `default:"200m"`
}
Request struct {
Mem string `default:"200Mi"`
Cpu string `default:"50m"`
}
}
ConfigReloaderCPU string
ConfigReloaderMemory string
}{ConfigReloadImage: "prometheus-config-reloader:latest"}),
},
},
want: []string{
"--reload-url=http://localhost:8429/-/reload",
Expand All @@ -959,7 +997,45 @@ func Test_buildConfigReloaderArgs(t *testing.T) {
cr: &victoriametricsv1beta1.VMAgent{
Spec: victoriametricsv1beta1.VMAgentSpec{Port: "8429"},
},
reloaderImage: "quay.io/coreos/prometheus-config-reloader:v0.42.0",
c: &config.BaseOperatorConf{
VMAgentDefault: struct {
Image string `default:"victoriametrics/vmagent"`
Version string `default:"v1.58.0"`
ConfigReloadImage string `default:"quay.io/prometheus-operator/prometheus-config-reloader:v0.48.1"`
Port string `default:"8429"`
UseDefaultResources bool `default:"true"`
Resource struct {
Limit struct {
Mem string `default:"500Mi"`
Cpu string `default:"200m"`
}
Request struct {
Mem string `default:"200Mi"`
Cpu string `default:"50m"`
}
}
ConfigReloaderCPU string `default:"100m"`
ConfigReloaderMemory string `default:"25Mi"`
}(struct {
Image string
Version string
ConfigReloadImage string
Port string
UseDefaultResources bool
Resource struct {
Limit struct {
Mem string `default:"500Mi"`
Cpu string `default:"200m"`
}
Request struct {
Mem string `default:"200Mi"`
Cpu string `default:"50m"`
}
}
ConfigReloaderCPU string
ConfigReloaderMemory string
}{ConfigReloadImage: "quay.io/coreos/prometheus-config-reloader:v0.42.0"}),
},
},
want: []string{
"--reload-url=http://localhost:8429/-/reload",
Expand All @@ -971,8 +1047,9 @@ func Test_buildConfigReloaderArgs(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := buildConfigReloaderArgs(tt.args.cr, tt.args.reloaderImage)

got := buildConfigReloaderArgs(tt.args.cr, tt.args.c)
sort.Strings(got)
sort.Strings(tt.want)
assert.Equal(t, tt.want, got)
})
}
Expand Down
Loading

0 comments on commit 8f6f133

Please sign in to comment.