Skip to content

Commit

Permalink
feat: etcd support (#65)
Browse files Browse the repository at this point in the history
* improvement: refactor for etcd support

* makefile: use gsed if using mac

* refactor upsert functions

* feat: create etcd instance for every vcluster created

* release: update version

* fix: issue with helm chart gen through make
  • Loading branch information
waveywaves authored Oct 24, 2023
1 parent 6ee1c37 commit e66b443
Show file tree
Hide file tree
Showing 34 changed files with 2,310 additions and 286 deletions.
26 changes: 18 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
#include tests/e2e/Makefile

VERSION ?= 1.3.8
VERSION ?= 1.4.0

# check if we are using MacOS or LINUX and use that to determine the sed command
UNAME_S := $(shell uname -s)
SED := sed
ifeq ($(UNAME_S),Darwin)
SED = gsed
else
SED = sed
endif


# CHANNELS define the bundle channels used in the bundle.
# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
Expand Down Expand Up @@ -135,7 +145,7 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
.PHONY: docker-buildx
docker-buildx: test ## Build and push docker image for the manager for cross-platform support
# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
$(SED) -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
- docker buildx create --name project-v3-builder
docker buildx use project-v3-builder
- docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
Expand Down Expand Up @@ -169,14 +179,14 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
build-helm-chart: manifests generate fmt vet kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
# update the crd
$(KUSTOMIZE) build config/crd > chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml
sed -i'' -e 's/labels:/labels: {{ include "common.labels.standard" . | nindent 4 }}/' chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml
$(SED) -i'' -e 's/labels:/labels: {{ include "common.labels.standard" . | nindent 4 }}/' chart/templates/uffizziclusters.uffizzi.com_customresourcedefinition.yaml
# update roles
cp config/rbac/role.yaml chart/templates/manager-role_clusterrole.yaml
sed -i'' -e 's/labels:/labels: {{ include "common.labels.standard" . | nindent 4 }}/' chart/templates/manager-role_clusterrole.yaml
sed -i'' -e 's/apiVersion: rbac.authorization.k8s.io\/v1/apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}/' chart/templates/manager-role_clusterrole.yaml
sed -i'' -e '/creationTimestamp: null/d' chart/templates/manager-role_clusterrole.yaml
sed -i'' -e 's/name: manager-role/name: {{ include "common.names.fullname" . }}-manager-role/' chart/templates/manager-role_clusterrole.yaml
sed -i'' -e '/metadata:/a\
$(SED) -i'' -e '/creationTimestamp: null/d' chart/templates/manager-role_clusterrole.yaml
$(SED) -i'' -e 's/name: manager-role/name: {{ include "common.names.fullname" . }}-manager-role/' chart/templates/manager-role_clusterrole.yaml
$(SED) -i'' -e 's/apiVersion: rbac.authorization.k8s.io\/v1/apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}/' chart/templates/manager-role_clusterrole.yaml
$(SED) -i'' -e 's/labels:/labels: {{ include "common.labels.standard" . | nindent 4 }}/' chart/templates/manager-role_clusterrole.yaml
$(SED) -i'' -e '/metadata:/a\
labels: {{ include "common.labels.standard" . | nindent 4 }}\
app.kubernetes.io/component: rbac\
app.kubernetes.io/part-of: uffizzi' chart/templates/manager-role_clusterrole.yaml
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha1/uffizzicluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,8 @@ type UffizziClusterResourceCount struct {

// UffizziClusterSpec defines the desired state of UffizziCluster
type UffizziClusterSpec struct {
//+kubebuilder:default:="k3s"
//+kubebuilder:validation:Enum=k3s;k8s
Distro string `json:"distro,omitempty"`
APIServer UffizziClusterAPIServer `json:"apiServer,omitempty"`
Ingress UffizziClusterIngress `json:"ingress,omitempty"`
Expand All @@ -150,6 +152,9 @@ type UffizziClusterSpec struct {
ResourceQuota *UffizziClusterResourceQuota `json:"resourceQuota,omitempty"`
LimitRange *UffizziClusterLimitRange `json:"limitRange,omitempty"`
Sleep bool `json:"sleep,omitempty"`
//+kubebuilder:default:="etcd"
//+kubebuilder:validation:Enum=etcd;sqlite
ExternalDatastore string `json:"externalDatastore,omitempty"`
}

// UffizziClusterStatus defines the observed state of UffizziCluster
Expand Down
4 changes: 2 additions & 2 deletions chart/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.3.8
version: 1.4.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v1.3.8"
appVersion: "v1.4.0"
dependencies:
- name: common
repository: https://charts.bitnami.com/bitnami
Expand Down
4 changes: 3 additions & 1 deletion chart/templates/manager-role_clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}
kind: ClusterRole
metadata:
labels: {{ include "common.labels.standard" . | nindent 4 }} app.kubernetes.io/component: rbac app.kubernetes.io/part-of: uffizzi
labels: {{ include "common.labels.standard" . | nindent 4 }}
app.kubernetes.io/component: rbac
app.kubernetes.io/part-of: uffizzi
name: {{ include "common.names.fullname" . }}-manager-role
rules:
- apiGroups:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,16 @@ spec:
type: string
type: object
distro:
default: k3s
enum:
- k3s
- k8s
type: string
externalDatastore:
default: etcd
enum:
- etcd
- sqlite
type: string
helm:
items:
Expand Down
2 changes: 1 addition & 1 deletion chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

image:
repository: docker.io/uffizzi/uffizzi-cluster-operator
tag: v1.3.8
tag: v1.4.0
# `flux` dependency values
flux:
helmController:
Expand Down
10 changes: 10 additions & 0 deletions config/crd/bases/uffizzi.com_uffizziclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,16 @@ spec:
type: string
type: object
distro:
default: k3s
enum:
- k3s
- k8s
type: string
externalDatastore:
default: etcd
enum:
- etcd
- sqlite
type: string
helm:
items:
Expand Down
22 changes: 22 additions & 0 deletions controllers/constants/constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package constants

const (
UFFIZZI_APP_COMPONENT_LABEL = "app.uffizzi.com/component"
VCLUSTER = "vcluster"
ETCD = "etcd"
UCLUSTER_NAME_PREFIX = "uc-"
LOFT_HELM_REPO = "loft"
BITNAMI_HELM_REPO = "bitnami"
VCLUSTER_CHART_K3S = "vcluster"
VCLUSTER_CHART_K8S = "vcluster-k8s"
VCLUSTER_CHART_K3S_VERSION = "0.16.4"
VCLUSTER_CHART_K8S_VERSION = "0.16.4"
ETCD_CHART = "etcd"
ETCD_CHART_VERSION = "9.5.6"
BITNAMI_CHART_REPO_URL = "oci://registry-1.docker.io/bitnamicharts"
LOFT_CHART_REPO_URL = "https://charts.loft.sh"
VCLUSTER_K3S_DISTRO = "k3s"
VCLUSTER_K8S_DISTRO = "k8s"
K3S_DATASTORE_ENDPOINT = "K3S_DATASTORE_ENDPOINT"
VCLUSTER_INGRESS_HOSTNAME = "VCLUSTER_INGRESS_HOST"
)
131 changes: 131 additions & 0 deletions controllers/etcd/etcd_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package uffizzicluster

import (
"context"
uclusteruffizzicomv1alpha1 "github.com/UffizziCloud/uffizzi-cluster-operator/api/v1alpha1"
"github.com/UffizziCloud/uffizzi-cluster-operator/controllers/constants"
fluxhelmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
controllerruntimesource "sigs.k8s.io/controller-runtime/pkg/source"
)

// UffizziClusterReconciler reconciles a UffizziCluster object
type UffizziClusterEtcdReconciler struct {
client.Client
Scheme *runtime.Scheme
}

func (r *UffizziClusterEtcdReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// check if the cluster is a k3s cluster
// if it is, then we need to create the etcd cluster
// default lifecycle operation
logger := log.FromContext(ctx)

// ----------------------
// UCLUSTER FIND AND CHECK
// ----------------------
// Fetch the UffizziCluster instance in question and then see which kind of event might have been triggered
uCluster := &uclusteruffizzicomv1alpha1.UffizziCluster{}
if err := r.Get(ctx, req.NamespacedName, uCluster); err != nil {
// Handle error
return ctrl.Result{}, client.IgnoreNotFound(err)
}

if uCluster.Spec.ExternalDatastore == constants.ETCD {
err := r.createBitnamiHelmRepo(ctx, req)
if err != nil && k8serrors.IsAlreadyExists(err) {
// logger.Info("Loft Helm Repo for UffizziCluster already exists", "NamespacedName", req.NamespacedName)
} else {
logger.Info("Bitnami Helm Repo for UffizziCluster created", "NamespacedName", req.NamespacedName)
}
// create a helm release for the etcd cluster
// check if the helm release exists
helmRelease := &fluxhelmv2beta1.HelmRelease{}
err = r.Get(ctx, client.ObjectKey{
Namespace: uCluster.Namespace,
Name: BuildEtcdHelmReleaseName(uCluster),
}, helmRelease)
if err != nil {
// if the helm release does not exist, create it
hr := buildEtcdHelmRelease(uCluster)
// add UffizziCluster as the owner of this HelmRelease
if err := ctrl.SetControllerReference(uCluster, hr, r.Scheme); err != nil {
logger.Error(err, "unable to set the controller reference")
return ctrl.Result{}, err
}
if err := r.Create(ctx, hr); err != nil {
logger.Error(err, "unable to create the helm release")
return ctrl.Result{}, err
}
} else {
return ctrl.Result{}, nil
}
}

return ctrl.Result{}, nil
}

func buildEtcdHelmRelease(uCluster *uclusteruffizzicomv1alpha1.UffizziCluster) *fluxhelmv2beta1.HelmRelease {
return &fluxhelmv2beta1.HelmRelease{
ObjectMeta: metav1.ObjectMeta{
Name: BuildEtcdHelmReleaseName(uCluster),
Namespace: uCluster.Namespace,
Labels: map[string]string{
constants.UFFIZZI_APP_COMPONENT_LABEL: constants.ETCD,
},
},
Spec: fluxhelmv2beta1.HelmReleaseSpec{
ReleaseName: BuildEtcdHelmReleaseName(uCluster),
Chart: fluxhelmv2beta1.HelmChartTemplate{
Spec: fluxhelmv2beta1.HelmChartTemplateSpec{
Chart: constants.ETCD_CHART,
Version: constants.ETCD_CHART_VERSION,
SourceRef: fluxhelmv2beta1.CrossNamespaceObjectReference{
Kind: "HelmRepository",
Name: constants.BITNAMI_HELM_REPO,
},
},
},
},
}
}

func BuildEtcdHelmReleaseName(uCluster *uclusteruffizzicomv1alpha1.UffizziCluster) string {
return constants.UCLUSTER_NAME_PREFIX + constants.ETCD + "-" + uCluster.Name
}

// SetupWithManager sets up the controller with the Manager.
func (r *UffizziClusterEtcdReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&uclusteruffizzicomv1alpha1.UffizziCluster{}).
// Watch HelmRelease reconciled by the Helm Controller
Watches(
&controllerruntimesource.Kind{Type: &fluxhelmv2beta1.HelmRelease{}},
&handler.EnqueueRequestForOwner{
IsController: true,
OwnerType: &uclusteruffizzicomv1alpha1.UffizziCluster{},
}).
Complete(r)
}
29 changes: 29 additions & 0 deletions controllers/etcd/helm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package uffizzicluster

import (
"context"
"github.com/UffizziCloud/uffizzi-cluster-operator/controllers/constants"
fluxsourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
ctrl "sigs.k8s.io/controller-runtime"
)

func (r *UffizziClusterEtcdReconciler) createHelmRepo(ctx context.Context, name, namespace, url string) error {
// Create HelmRepository in the same namespace as the HelmRelease
helmRepo := &fluxsourcev1.HelmRepository{
ObjectMeta: ctrl.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: fluxsourcev1.HelmRepositorySpec{
URL: url,
Type: "oci",
},
}

err := r.Create(ctx, helmRepo)
return err
}

func (r *UffizziClusterEtcdReconciler) createBitnamiHelmRepo(ctx context.Context, req ctrl.Request) error {
return r.createHelmRepo(ctx, constants.BITNAMI_HELM_REPO, req.Namespace, constants.BITNAMI_CHART_REPO_URL)
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package controllers
package uffizzicluster

import (
uclusteruffizzicomv1alpha1 "github.com/UffizziCloud/uffizzi-cluster-operator/api/v1alpha1"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package controllers
package uffizzicluster

import (
"testing"
Expand Down
Loading

0 comments on commit e66b443

Please sign in to comment.