From 938a0839eab13557408a0ac527740502f164c600 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Mon, 23 Dec 2024 02:23:05 +0900 Subject: [PATCH 01/23] Add cluster in e2e --- .github/cluster.yml | 372 +++++++++++++++++++++++++++++++++++++++++++ test/e2e/e2e_test.go | 9 ++ 2 files changed, 381 insertions(+) create mode 100644 .github/cluster.yml diff --git a/.github/cluster.yml b/.github/cluster.yml new file mode 100644 index 0000000..3cba479 --- /dev/null +++ b/.github/cluster.yml @@ -0,0 +1,372 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: quick-start +spec: + clusterNetwork: + services: + cidrBlocks: 10.128.0.0/12 + pods: + cidrBlocks: "192.168.0.0/16" + serviceDomain: "cluster.local" + topology: + class: quick-start + controlPlane: + metadata: {} + replicas: 1 + variables: + - name: imageRepository + value: "" + - name: etcdImageTag + value: "" + - name: coreDNSImageTag + value: "" + - name: podSecurityStandard + value: + enabled: true + enforce: "baseline" + audit: "restricted" + warn: "restricted" + version: v1.31.1 + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 1 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: quick-start +spec: + controlPlane: + machineHealthCheck: + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready + machineInfrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IncusMachineTemplate + name: quick-start-control-plane + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: quick-start-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IncusClusterTemplate + name: quick-start-cluster + patches: + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository + valueFrom: + variable: imageRepository + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + description: Sets the imageRepository used for the KubeadmControlPlane. + enabledIf: '{{ ne .imageRepository "" }}' + name: imageRepository + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd + valueFrom: + template: | + local: + imageTag: {{ .etcdImageTag }} + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + description: Sets tag to use for the etcd image in the KubeadmControlPlane. + name: etcdImageTag + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns + valueFrom: + template: | + imageTag: {{ .coreDNSImageTag }} + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + description: Sets tag to use for the etcd image in the KubeadmControlPlane. + name: coreDNSImageTag + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/customImage + valueFrom: + template: | + kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }} + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IncusMachineTemplate + matchResources: + machineDeploymentClass: + names: + - default-worker + - jsonPatches: + - op: add + path: /spec/template/spec/customImage + valueFrom: + template: | + kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }} + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IncusMachineTemplate + matchResources: + controlPlane: true + description: Sets the container image that is used for running IncusMachines + for the controlPlane and default-worker machineDeployments. + name: customImage + - definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs + value: + admission-control-config-file: /etc/kubernetes/kube-apiserver-admission-pss.yaml + - op: add + path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes + value: + - hostPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml + mountPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml + name: admission-pss + pathType: File + readOnly: true + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files + valueFrom: + template: | + - content: | + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1{{ if semverCompare "< v1.25" .builtin.controlPlane.version }}beta1{{ end }} + kind: PodSecurityConfiguration + defaults: + enforce: "{{ .podSecurityStandard.enforce }}" + enforce-version: "latest" + audit: "{{ .podSecurityStandard.audit }}" + audit-version: "latest" + warn: "{{ .podSecurityStandard.warn }}" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [kube-system] + path: /etc/kubernetes/kube-apiserver-admission-pss.yaml + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + description: Adds an admission configuration for PodSecurity to the kube-apiserver. + enabledIf: '{{ .podSecurityStandard.enabled }}' + name: podSecurityStandard + variables: + - name: imageRepository + required: true + schema: + openAPIV3Schema: + default: "" + description: imageRepository sets the container registry to pull images from. + If empty, nothing will be set and the from of kubeadm will be used. + example: registry.k8s.io + type: string + - name: etcdImageTag + required: true + schema: + openAPIV3Schema: + default: "" + description: etcdImageTag sets the tag for the etcd image. + example: 3.5.3-0 + type: string + - name: coreDNSImageTag + required: true + schema: + openAPIV3Schema: + default: "" + description: coreDNSImageTag sets the tag for the coreDNS image. + example: v1.8.5 + type: string + - name: podSecurityStandard + required: false + schema: + openAPIV3Schema: + properties: + audit: + default: restricted + description: audit sets the level for the audit PodSecurityConfiguration + mode. One of privileged, baseline, restricted. + type: string + enabled: + default: true + description: enabled enables the patches to enable Pod Security Standard + via AdmissionConfiguration. + type: boolean + enforce: + default: baseline + description: enforce sets the level for the enforce PodSecurityConfiguration + mode. One of privileged, baseline, restricted. + type: string + warn: + default: restricted + description: warn sets the level for the warn PodSecurityConfiguration + mode. One of privileged, baseline, restricted. + type: string + type: object + workers: + machineDeployments: + - class: default-worker + machineHealthCheck: + unhealthyConditions: + - status: Unknown + timeout: 300s + type: Ready + - status: "False" + timeout: 300s + type: Ready + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: quick-start-default-worker-bootstraptemplate + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: IncusMachineTemplate + name: quick-start-default-worker-machinetemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IncusClusterTemplate +metadata: + name: quick-start-cluster +spec: + template: + spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: quick-start-control-plane +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + - 0.0.0.0 + - host.docker.internal + initConfiguration: + nodeRegistration: {} + joinConfiguration: + nodeRegistration: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IncusMachineTemplate +metadata: + name: quick-start-control-plane +spec: + template: + spec: + instanceSpec: + type: container + source: + alias: ubuntu/24.04/cloud + config: + raw.lxc: >- + lxc.apparmor.profile=unconfined\nlxc.cap.drop=\nlxc.cgroup.devices.allow=a\nlxc.mount.auto=proc:rw + sys:rw + security.nesting: 'true' + security.privileged: 'true' + profiles: + - default +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: IncusMachineTemplate +metadata: + name: quick-start-default-worker-machinetemplate +spec: + template: + spec: + instanceSpec: + type: container + source: + alias: ubuntu/24.04/cloud + config: + raw.lxc: >- + lxc.apparmor.profile=unconfined\nlxc.cap.drop=\nlxc.cgroup.devices.allow=a\nlxc.mount.auto=proc:rw + sys:rw + security.nesting: 'true' + security.privileged: 'true' + profiles: + - default +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: quick-start-default-worker-bootstraptemplate +spec: + template: + spec: + joinConfiguration: + nodeRegistration: {} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + services: + cidrBlocks: + - 10.128.0.0/12 + topology: + class: quick-start + controlPlane: + metadata: {} + replicas: 1 + variables: + - name: imageRepository + value: "" + - name: etcdImageTag + value: "" + - name: coreDNSImageTag + value: "" + - name: podSecurityStandard + value: + audit: restricted + enabled: true + enforce: baseline + warn: restricted + version: v1.31.1 + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 0 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e35a800..1e63de4 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -291,6 +291,15 @@ var _ = Describe("Manager", Ordered, func() { // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, // strings.ToLower(), // )) + + It("create an IncusCluster", func() { + By("creating a ClusterClass/Cluster/IncusCluster/etc for the IncusCluster") + cmd := exec.Command("kubectl", "apply", "-n", namespace, + "-f", ".github/cluster.yml", + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create Cluster") + }) }) }) From 77ed0749ee62b3a9eb6f643050548475e3923cc5 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Mon, 23 Dec 2024 02:25:15 +0900 Subject: [PATCH 02/23] Enable ts --- .github/workflows/test-e2e.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 9f58b79..727d408 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -16,6 +16,22 @@ jobs: - name: Clone the code uses: actions/checkout@v4 + - name: Tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + - name: Install and Run SSH server + run: | + sudo apt-get update + sudo apt-get install -y openssh-server + sudo systemctl start ssh + sudo systemctl status ssh + whoami + mkdir -p ~/.ssh + curl -L https://github.com/tsuzu.keys >> ~/.ssh/authorized_keys + - run: | cd .github && docker compose up -d docker ps -a @@ -94,3 +110,9 @@ jobs: export INCUS_TOKEN=~/.config/incus/oidctokens/incus.txt go mod tidy make test-e2e + + - name: Wait + if: failure() + run: | + sleep 3600 + echo "Failed" From 2912cbda218c15dcc9d65611d50a2f23a73d89ab Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Mon, 23 Dec 2024 02:54:14 +0900 Subject: [PATCH 03/23] Fix apiVersion --- .github/cluster.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/cluster.yml b/.github/cluster.yml index 3cba479..18d80ab 100644 --- a/.github/cluster.yml +++ b/.github/cluster.yml @@ -50,7 +50,7 @@ spec: type: Ready machineInfrastructure: ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate name: quick-start-control-plane ref: @@ -59,7 +59,7 @@ spec: name: quick-start-control-plane infrastructure: ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusClusterTemplate name: quick-start-cluster patches: @@ -114,7 +114,7 @@ spec: template: | kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }} selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate matchResources: machineDeploymentClass: @@ -127,7 +127,7 @@ spec: template: | kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }} selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate matchResources: controlPlane: true @@ -251,11 +251,11 @@ spec: name: quick-start-default-worker-bootstraptemplate infrastructure: ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate name: quick-start-default-worker-machinetemplate --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusClusterTemplate metadata: name: quick-start-cluster @@ -283,7 +283,7 @@ spec: joinConfiguration: nodeRegistration: {} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate metadata: name: quick-start-control-plane @@ -303,7 +303,7 @@ spec: profiles: - default --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: IncusMachineTemplate metadata: name: quick-start-default-worker-machinetemplate From 246f09dd07c93e9cb22ffe94389176b5287f7f10 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Mon, 23 Dec 2024 03:00:14 +0900 Subject: [PATCH 04/23] Fix --- .github/cluster.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/cluster.yml b/.github/cluster.yml index 18d80ab..a38232e 100644 --- a/.github/cluster.yml +++ b/.github/cluster.yml @@ -5,9 +5,11 @@ metadata: spec: clusterNetwork: services: - cidrBlocks: 10.128.0.0/12 + cidrBlocks: + - 10.128.0.0/12 pods: - cidrBlocks: "192.168.0.0/16" + cidrBlocks: + - "192.168.0.0/16" serviceDomain: "cluster.local" topology: class: quick-start From b3d715f38166aeac7fa5faab4a11e6802729cfcb Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Mon, 23 Dec 2024 03:24:06 +0900 Subject: [PATCH 05/23] debug --- .github/workflows/test-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 727d408..55e3314 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -110,6 +110,7 @@ jobs: export INCUS_TOKEN=~/.config/incus/oidctokens/incus.txt go mod tidy make test-e2e + exit 1 - name: Wait if: failure() From fa19126f0cfdc77b50baf7902f455374ee353ad5 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Tue, 31 Dec 2024 23:28:03 +0900 Subject: [PATCH 06/23] Fix bugs --- config/crd/kustomization.yaml | 3 +++ config/default/kustomization.yaml | 2 +- config/rbac/role.yaml | 9 +++++++++ internal/controller/incusmachine_controller.go | 6 +++++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 688df32..48a36a6 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,6 +8,9 @@ resources: - bases/infrastructure.cluster.x-k8s.io_incusmachinetemplates.yaml # +kubebuilder:scaffold:crdkustomizeresource +commonLabels: + cluster.x-k8s.io/v1beta1: v1alpha1 + patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 03407ce..00ef54a 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,5 +1,5 @@ # Adds namespace to all resources. -namespace: cluster-api-provider-incus-system +namespace: capincus-system # Value of this field is prepended to the # names of all resources, e.g. a deployment named diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e1654bd..4176a4f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,11 +4,20 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch - apiGroups: - cluster.x-k8s.io resources: - clusters - machines + - machinesets verbs: - get - list diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index d588d97..03aaf6b 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -55,6 +55,8 @@ type IncusMachineReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=incusmachines/finalizers,verbs=update // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines,verbs=get;list;watch +// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinesets,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -153,7 +155,9 @@ func (r *IncusMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request // Note: Finalizers in general can only be added when the deletionTimestamp is not set. if incusMachine.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) { controllerutil.AddFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) - return ctrl.Result{}, nil + return ctrl.Result{ + Requeue: true, + }, nil } // Handle deleted machines From ccadc573e848956a2a624202a98d0783a195fea6 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Tue, 31 Dec 2024 23:30:03 +0900 Subject: [PATCH 07/23] Add debug logs --- internal/controller/incusmachine_controller.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 03aaf6b..1fadc24 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -151,9 +151,13 @@ func (r *IncusMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request } }() + log.Info("Reconciling IncusMachine") + // Add finalizer first if not set to avoid the race condition between init and delete. // Note: Finalizers in general can only be added when the deletionTimestamp is not set. if incusMachine.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) { + log.Info("Adding finalizer for IncusMachine") + controllerutil.AddFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) return ctrl.Result{ Requeue: true, @@ -205,6 +209,7 @@ func (r *IncusMachineReconciler) reconcileDelete(ctx context.Context, _ *infrav1 output, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) if errors.Is(err, incus.ErrorInstanceNotFound) { // Instance is already deleted so remove the finalizer. + log.Info("Deleting finalizer from IncusMachine") controllerutil.RemoveFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) return ctrl.Result{}, nil } @@ -214,10 +219,14 @@ func (r *IncusMachineReconciler) reconcileDelete(ctx context.Context, _ *infrav1 if output.StatusCode != api.Stopped && output.StatusCode != api.Stopping { + log.Info("Stopping instance") + if err := r.IncusClient.StopInstance(ctx, incusMachine.Name); err != nil { log.Info("Failed to stop instance", "error", err) } } else if output.StatusCode != api.OperationCreated { + log.Info("Deleting instance") + if err := r.IncusClient.DeleteInstance(ctx, incusMachine.Name); err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete instance: %w", err) } @@ -257,6 +266,8 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c return ctrl.Result{}, err } + log.Info("Creating IncusMachine instance") + // Create the instance err = r.IncusClient.CreateInstance(ctx, incus.CreateInstanceInput{ Name: incusMachine.Name, From bf69be317643cb22d14a48da8fa9d90c062b5750 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 00:09:36 +0900 Subject: [PATCH 08/23] RequeueAfter --- internal/controller/incusmachine_controller.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 1fadc24..ce54a42 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/lxc/incus/shared/api" infrav1alpha1 "github.com/miscord-dev/cluster-api-provider-incus/api/v1alpha1" @@ -224,12 +225,20 @@ func (r *IncusMachineReconciler) reconcileDelete(ctx context.Context, _ *infrav1 if err := r.IncusClient.StopInstance(ctx, incusMachine.Name); err != nil { log.Info("Failed to stop instance", "error", err) } + + return ctrl.Result{ + RequeueAfter: 5 * time.Second, + }, nil } else if output.StatusCode != api.OperationCreated { log.Info("Deleting instance") if err := r.IncusClient.DeleteInstance(ctx, incusMachine.Name); err != nil { return ctrl.Result{}, fmt.Errorf("failed to delete instance: %w", err) } + + return ctrl.Result{ + RequeueAfter: 5 * time.Second, + }, nil } return ctrl.Result{ From e8286d1a4cfe10f3f60f19c1ecd253ead8e213c2 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 01:02:52 +0900 Subject: [PATCH 09/23] Add InstanceExists --- internal/controller/incusmachine_controller.go | 9 +++++++-- pkg/incus/incus.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index ce54a42..7c59834 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -207,13 +207,18 @@ func (r *IncusMachineReconciler) reconcileDelete(ctx context.Context, _ *infrav1 // return err // } - output, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) - if errors.Is(err, incus.ErrorInstanceNotFound) { + exists, err := r.IncusClient.InstanceExists(ctx, incusMachine.Name) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to check if instance exists: %w", err) + } + if !exists { // Instance is already deleted so remove the finalizer. log.Info("Deleting finalizer from IncusMachine") controllerutil.RemoveFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) return ctrl.Result{}, nil } + + output, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to get instance: %w", err) } diff --git a/pkg/incus/incus.go b/pkg/incus/incus.go index 537868f..df9fa01 100644 --- a/pkg/incus/incus.go +++ b/pkg/incus/incus.go @@ -37,6 +37,7 @@ type GetInstanceOutput struct { type Client interface { CreateInstance(ctx context.Context, spec CreateInstanceInput) error + InstanceExists(ctx context.Context, name string) (bool, error) GetInstance(ctx context.Context, name string) (*GetInstanceOutput, error) DeleteInstance(ctx context.Context, name string) error StopInstance(ctx context.Context, name string) error @@ -108,6 +109,19 @@ func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) e return nil } +func (c *client) InstanceExists(ctx context.Context, name string) (bool, error) { + _, _, err := c.client.GetInstance(name) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return false, nil + } + + return false, fmt.Errorf("failed to get instance: %w", err) + } + + return true, nil +} + func (c *client) GetInstance(ctx context.Context, name string) (*GetInstanceOutput, error) { resp, _, err := c.client.GetInstanceFull(name) if err != nil { From 9ae12bbcf7b651d208a73992e7cd9d02cd118661 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 02:20:53 +0900 Subject: [PATCH 10/23] nil check --- pkg/incus/incus.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/incus/incus.go b/pkg/incus/incus.go index df9fa01..0e3cba1 100644 --- a/pkg/incus/incus.go +++ b/pkg/incus/incus.go @@ -55,6 +55,9 @@ func NewClient(instanceServer incusclient.InstanceServer) Client { func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) error { config := maps.Clone(spec.Config) + if config == nil { + config = make(map[string]string) + } switch spec.BootstrapData.Format { case "cloud-config": From 1c794b2d7f9c2f30ea8b0401bdc2340eda075632 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 02:23:25 +0900 Subject: [PATCH 11/23] Log bootstrap data --- pkg/incus/incus.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/incus/incus.go b/pkg/incus/incus.go index 0e3cba1..c470e7c 100644 --- a/pkg/incus/incus.go +++ b/pkg/incus/incus.go @@ -9,6 +9,7 @@ import ( incusclient "github.com/lxc/incus/client" "github.com/lxc/incus/shared/api" infrav1alpha1 "github.com/miscord-dev/cluster-api-provider-incus/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" ) var ( @@ -54,6 +55,8 @@ func NewClient(instanceServer incusclient.InstanceServer) Client { } func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) error { + log := ctrl.LoggerFrom(ctx) + config := maps.Clone(spec.Config) if config == nil { config = make(map[string]string) @@ -68,6 +71,11 @@ func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) e return fmt.Errorf("unsupported bootstrap data format: %s", spec.BootstrapData.Format) } + log.Info("bootstrap data", + "format", spec.BootstrapData.Format, + "data_len", len(spec.BootstrapData.Data), + ) + req := api.InstancesPost{ Name: spec.Name, InstancePut: api.InstancePut{ From 8c1da4011d18725c8e54eae0ce1e84b3a7069e66 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 03:26:27 +0900 Subject: [PATCH 12/23] Fix a bug --- pkg/incus/incus.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pkg/incus/incus.go b/pkg/incus/incus.go index c470e7c..2ef35c7 100644 --- a/pkg/incus/incus.go +++ b/pkg/incus/incus.go @@ -9,7 +9,6 @@ import ( incusclient "github.com/lxc/incus/client" "github.com/lxc/incus/shared/api" infrav1alpha1 "github.com/miscord-dev/cluster-api-provider-incus/api/v1alpha1" - ctrl "sigs.k8s.io/controller-runtime" ) var ( @@ -55,8 +54,6 @@ func NewClient(instanceServer incusclient.InstanceServer) Client { } func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) error { - log := ctrl.LoggerFrom(ctx) - config := maps.Clone(spec.Config) if config == nil { config = make(map[string]string) @@ -71,16 +68,11 @@ func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) e return fmt.Errorf("unsupported bootstrap data format: %s", spec.BootstrapData.Format) } - log.Info("bootstrap data", - "format", spec.BootstrapData.Format, - "data_len", len(spec.BootstrapData.Data), - ) - req := api.InstancesPost{ Name: spec.Name, InstancePut: api.InstancePut{ Architecture: spec.Architecture, - Config: spec.Config, + Config: config, Devices: spec.Devices, Ephemeral: spec.Ephemeral, Profiles: spec.Profiles, From 6b6e7c0922028cc068ac346d9fb15ff9f1ee9a9c Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Wed, 1 Jan 2025 19:09:45 +0900 Subject: [PATCH 13/23] refactor --- .../controller/incusmachine_controller.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 7c59834..f39a884 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -142,6 +142,13 @@ func (r *IncusMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request if err != nil { return ctrl.Result{}, err } + deleted := !incusMachine.ObjectMeta.DeletionTimestamp.IsZero() + containsFinalizer := controllerutil.ContainsFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) + + if deleted && !containsFinalizer { + // Return early since the object is being deleted and doesn't have the finalizer. + return ctrl.Result{}, nil + } // Always attempt to Patch the IncusMachine object and status after each reconciliation. defer func() { if err := patchIncusMachine(ctx, patchHelper, incusMachine); err != nil { @@ -154,9 +161,14 @@ func (r *IncusMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request log.Info("Reconciling IncusMachine") + // Handle deleted machines + if deleted { + return r.reconcileDelete(ctx, incusCluster, machine, incusMachine) + } + // Add finalizer first if not set to avoid the race condition between init and delete. // Note: Finalizers in general can only be added when the deletionTimestamp is not set. - if incusMachine.ObjectMeta.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) { + if !containsFinalizer { log.Info("Adding finalizer for IncusMachine") controllerutil.AddFinalizer(incusMachine, infrav1alpha1.MachineFinalizer) @@ -165,11 +177,6 @@ func (r *IncusMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request }, nil } - // Handle deleted machines - if !incusMachine.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, incusCluster, machine, incusMachine) - } - // Handle non-deleted machines return r.reconcileNormal(ctx, cluster, incusCluster, machine, incusMachine) } From 60085ad165eb217441ea4b0869685d40b06e9cdf Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 00:45:23 +0900 Subject: [PATCH 14/23] Set machine.Ready to true --- internal/controller/incusmachine_controller.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index f39a884..8419974 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -274,8 +274,14 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c } dataSecretName := *machine.Spec.Bootstrap.DataSecretName - _, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) + output, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) if err == nil { + if r.isMachineReady(ctx, output) { + log.Info("IncusMachine instance is ready") + + incusMachine.Status.Ready = true + } + return ctrl.Result{}, nil } if !errors.Is(err, incus.ErrorInstanceNotFound) { @@ -325,6 +331,10 @@ func (r *IncusMachineReconciler) getBootstrapData(ctx context.Context, namespace return string(value), bootstrapv1.Format(format), nil } +func (r *IncusMachineReconciler) isMachineReady(ctx context.Context, output *incus.GetInstanceOutput) bool { + return output.StatusCode == api.Ready +} + // SetupWithManager sets up the controller with the Manager. func (r *IncusMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { clusterToIncusMachines, err := util.ClusterToTypedObjectsMapper(mgr.GetClient(), &infrav1alpha1.IncusMachineList{}, mgr.GetScheme()) From e293ae22d46f917f4fc7589992d51c95ac21142c Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 00:50:19 +0900 Subject: [PATCH 15/23] Fix isMachineReady --- internal/controller/incusmachine_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 8419974..be99abc 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -332,7 +332,7 @@ func (r *IncusMachineReconciler) getBootstrapData(ctx context.Context, namespace } func (r *IncusMachineReconciler) isMachineReady(ctx context.Context, output *incus.GetInstanceOutput) bool { - return output.StatusCode == api.Ready + return output.StatusCode == api.Running } // SetupWithManager sets up the controller with the Manager. From f68a253d86d479c8cf638e0dfcde217bae6ff057 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:00:17 +0900 Subject: [PATCH 16/23] Requeue if machine isn't ready --- internal/controller/incusmachine_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index be99abc..52b24b1 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -280,9 +280,13 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c log.Info("IncusMachine instance is ready") incusMachine.Status.Ready = true + + return ctrl.Result{}, nil } - return ctrl.Result{}, nil + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil } if !errors.Is(err, incus.ErrorInstanceNotFound) { return ctrl.Result{}, fmt.Errorf("failed to get instance: %w", err) From dcacea5c714eddcf0675d35458ac61b71e5d235d Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:40:25 +0900 Subject: [PATCH 17/23] Set providerID --- internal/controller/incusmachine_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 52b24b1..8cd613c 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -29,6 +29,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -279,6 +280,7 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c if r.isMachineReady(ctx, output) { log.Info("IncusMachine instance is ready") + incusMachine.Spec.ProviderID = ptr.To("incus://" + output.Name) incusMachine.Status.Ready = true return ctrl.Result{}, nil From 0f253b5eb59e4a7a9801f688f817cae1066dac61 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 02:33:33 +0900 Subject: [PATCH 18/23] Embeds providerID --- go.mod | 2 +- internal/controller/incusmachine_controller.go | 12 ++++++++++-- pkg/incus/incus.go | 14 +++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6c316e3..06c48c4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/miscord-dev/cluster-api-provider-incus go 1.22.0 require ( + github.com/google/uuid v1.6.0 github.com/lxc/incus v0.7.0 github.com/onsi/ginkgo/v2 v2.19.1 github.com/onsi/gomega v1.34.0 @@ -43,7 +44,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/schema v1.2.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/websocket v1.5.1 // indirect diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 8cd613c..77e55b2 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/lxc/incus/shared/api" infrav1alpha1 "github.com/miscord-dev/cluster-api-provider-incus/api/v1alpha1" "github.com/miscord-dev/cluster-api-provider-incus/pkg/incus" @@ -29,7 +30,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" - "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -280,7 +280,7 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c if r.isMachineReady(ctx, output) { log.Info("IncusMachine instance is ready") - incusMachine.Spec.ProviderID = ptr.To("incus://" + output.Name) + incusMachine.Spec.ProviderID = &output.ProviderID incusMachine.Status.Ready = true return ctrl.Result{}, nil @@ -301,6 +301,13 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c log.Info("Creating IncusMachine instance") + uuid, err := uuid.NewV7() + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to generate UUID: %w", err) + } + + providerID := fmt.Sprintf("incus://%s", uuid.String()) + // Create the instance err = r.IncusClient.CreateInstance(ctx, incus.CreateInstanceInput{ Name: incusMachine.Name, @@ -308,6 +315,7 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c Data: bootstrapData, Format: string(bootstrapFormat), }, + ProviderID: providerID, InstanceSpec: incusMachine.Spec.InstanceSpec, }) if err != nil { diff --git a/pkg/incus/incus.go b/pkg/incus/incus.go index 2ef35c7..4438295 100644 --- a/pkg/incus/incus.go +++ b/pkg/incus/incus.go @@ -15,6 +15,12 @@ var ( ErrorInstanceNotFound = fmt.Errorf("instance not found") ) +const ( + configUserDataKey = "cloud-init.user-data" + configProviderIDKey = "user.provider-id" + configMetaDataKey = "user.meta-data" +) + type BootstrapData struct { Format string Data string @@ -23,6 +29,7 @@ type BootstrapData struct { type CreateInstanceInput struct { Name string BootstrapData BootstrapData + ProviderID string infrav1alpha1.InstanceSpec } @@ -31,6 +38,7 @@ type GetInstanceOutput struct { Name string infrav1alpha1.InstanceSpec + ProviderID string // TODO: Add status StatusCode api.StatusCode } @@ -61,13 +69,16 @@ func (c *client) CreateInstance(ctx context.Context, spec CreateInstanceInput) e switch spec.BootstrapData.Format { case "cloud-config": - config["cloud-init.user-data"] = spec.BootstrapData.Data + config[configUserDataKey] = spec.BootstrapData.Data case "": // Do nothing default: return fmt.Errorf("unsupported bootstrap data format: %s", spec.BootstrapData.Format) } + config[configProviderIDKey] = spec.ProviderID + config[configMetaDataKey] = fmt.Sprintf("provider-id: %s", spec.ProviderID) + req := api.InstancesPost{ Name: spec.Name, InstancePut: api.InstancePut{ @@ -148,6 +159,7 @@ func (c *client) GetInstance(ctx context.Context, name string) (*GetInstanceOutp Description: resp.Description, Type: infrav1alpha1.InstanceType(resp.Type), }, + ProviderID: resp.Config[configProviderIDKey], StatusCode: resp.StatusCode, }, nil } From e6f0ae41377dacfdd53801f54839b7a126a4424a Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 03:06:44 +0900 Subject: [PATCH 19/23] Delete .github/cluster.yml --- .github/cluster.yml | 374 -------------------------------------------- 1 file changed, 374 deletions(-) delete mode 100644 .github/cluster.yml diff --git a/.github/cluster.yml b/.github/cluster.yml deleted file mode 100644 index a38232e..0000000 --- a/.github/cluster.yml +++ /dev/null @@ -1,374 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: quick-start -spec: - clusterNetwork: - services: - cidrBlocks: - - 10.128.0.0/12 - pods: - cidrBlocks: - - "192.168.0.0/16" - serviceDomain: "cluster.local" - topology: - class: quick-start - controlPlane: - metadata: {} - replicas: 1 - variables: - - name: imageRepository - value: "" - - name: etcdImageTag - value: "" - - name: coreDNSImageTag - value: "" - - name: podSecurityStandard - value: - enabled: true - enforce: "baseline" - audit: "restricted" - warn: "restricted" - version: v1.31.1 - workers: - machineDeployments: - - class: default-worker - name: md-0 - replicas: 1 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: ClusterClass -metadata: - name: quick-start -spec: - controlPlane: - machineHealthCheck: - unhealthyConditions: - - status: Unknown - timeout: 300s - type: Ready - - status: "False" - timeout: 300s - type: Ready - machineInfrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: IncusMachineTemplate - name: quick-start-control-plane - ref: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - name: quick-start-control-plane - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: IncusClusterTemplate - name: quick-start-cluster - patches: - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository - valueFrom: - variable: imageRepository - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - description: Sets the imageRepository used for the KubeadmControlPlane. - enabledIf: '{{ ne .imageRepository "" }}' - name: imageRepository - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/etcd - valueFrom: - template: | - local: - imageTag: {{ .etcdImageTag }} - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - description: Sets tag to use for the etcd image in the KubeadmControlPlane. - name: etcdImageTag - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/dns - valueFrom: - template: | - imageTag: {{ .coreDNSImageTag }} - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - description: Sets tag to use for the etcd image in the KubeadmControlPlane. - name: coreDNSImageTag - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/customImage - valueFrom: - template: | - kindest/node:{{ .builtin.machineDeployment.version | replace "+" "_" }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: IncusMachineTemplate - matchResources: - machineDeploymentClass: - names: - - default-worker - - jsonPatches: - - op: add - path: /spec/template/spec/customImage - valueFrom: - template: | - kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: IncusMachineTemplate - matchResources: - controlPlane: true - description: Sets the container image that is used for running IncusMachines - for the controlPlane and default-worker machineDeployments. - name: customImage - - definitions: - - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs - value: - admission-control-config-file: /etc/kubernetes/kube-apiserver-admission-pss.yaml - - op: add - path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraVolumes - value: - - hostPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml - mountPath: /etc/kubernetes/kube-apiserver-admission-pss.yaml - name: admission-pss - pathType: File - readOnly: true - - op: add - path: /spec/template/spec/kubeadmConfigSpec/files - valueFrom: - template: | - - content: | - apiVersion: apiserver.config.k8s.io/v1 - kind: AdmissionConfiguration - plugins: - - name: PodSecurity - configuration: - apiVersion: pod-security.admission.config.k8s.io/v1{{ if semverCompare "< v1.25" .builtin.controlPlane.version }}beta1{{ end }} - kind: PodSecurityConfiguration - defaults: - enforce: "{{ .podSecurityStandard.enforce }}" - enforce-version: "latest" - audit: "{{ .podSecurityStandard.audit }}" - audit-version: "latest" - warn: "{{ .podSecurityStandard.warn }}" - warn-version: "latest" - exemptions: - usernames: [] - runtimeClasses: [] - namespaces: [kube-system] - path: /etc/kubernetes/kube-apiserver-admission-pss.yaml - selector: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlaneTemplate - matchResources: - controlPlane: true - description: Adds an admission configuration for PodSecurity to the kube-apiserver. - enabledIf: '{{ .podSecurityStandard.enabled }}' - name: podSecurityStandard - variables: - - name: imageRepository - required: true - schema: - openAPIV3Schema: - default: "" - description: imageRepository sets the container registry to pull images from. - If empty, nothing will be set and the from of kubeadm will be used. - example: registry.k8s.io - type: string - - name: etcdImageTag - required: true - schema: - openAPIV3Schema: - default: "" - description: etcdImageTag sets the tag for the etcd image. - example: 3.5.3-0 - type: string - - name: coreDNSImageTag - required: true - schema: - openAPIV3Schema: - default: "" - description: coreDNSImageTag sets the tag for the coreDNS image. - example: v1.8.5 - type: string - - name: podSecurityStandard - required: false - schema: - openAPIV3Schema: - properties: - audit: - default: restricted - description: audit sets the level for the audit PodSecurityConfiguration - mode. One of privileged, baseline, restricted. - type: string - enabled: - default: true - description: enabled enables the patches to enable Pod Security Standard - via AdmissionConfiguration. - type: boolean - enforce: - default: baseline - description: enforce sets the level for the enforce PodSecurityConfiguration - mode. One of privileged, baseline, restricted. - type: string - warn: - default: restricted - description: warn sets the level for the warn PodSecurityConfiguration - mode. One of privileged, baseline, restricted. - type: string - type: object - workers: - machineDeployments: - - class: default-worker - machineHealthCheck: - unhealthyConditions: - - status: Unknown - timeout: 300s - type: Ready - - status: "False" - timeout: 300s - type: Ready - template: - bootstrap: - ref: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: quick-start-default-worker-bootstraptemplate - infrastructure: - ref: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 - kind: IncusMachineTemplate - name: quick-start-default-worker-machinetemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: IncusClusterTemplate -metadata: - name: quick-start-cluster -spec: - template: - spec: {} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlaneTemplate -metadata: - name: quick-start-control-plane -spec: - template: - spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - certSANs: - - localhost - - 127.0.0.1 - - 0.0.0.0 - - host.docker.internal - initConfiguration: - nodeRegistration: {} - joinConfiguration: - nodeRegistration: {} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: IncusMachineTemplate -metadata: - name: quick-start-control-plane -spec: - template: - spec: - instanceSpec: - type: container - source: - alias: ubuntu/24.04/cloud - config: - raw.lxc: >- - lxc.apparmor.profile=unconfined\nlxc.cap.drop=\nlxc.cgroup.devices.allow=a\nlxc.mount.auto=proc:rw - sys:rw - security.nesting: 'true' - security.privileged: 'true' - profiles: - - default ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 -kind: IncusMachineTemplate -metadata: - name: quick-start-default-worker-machinetemplate -spec: - template: - spec: - instanceSpec: - type: container - source: - alias: ubuntu/24.04/cloud - config: - raw.lxc: >- - lxc.apparmor.profile=unconfined\nlxc.cap.drop=\nlxc.cgroup.devices.allow=a\nlxc.mount.auto=proc:rw - sys:rw - security.nesting: 'true' - security.privileged: 'true' - profiles: - - default ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: quick-start-default-worker-bootstraptemplate -spec: - template: - spec: - joinConfiguration: - nodeRegistration: {} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: test -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - serviceDomain: cluster.local - services: - cidrBlocks: - - 10.128.0.0/12 - topology: - class: quick-start - controlPlane: - metadata: {} - replicas: 1 - variables: - - name: imageRepository - value: "" - - name: etcdImageTag - value: "" - - name: coreDNSImageTag - value: "" - - name: podSecurityStandard - value: - audit: restricted - enabled: true - enforce: baseline - warn: restricted - version: v1.31.1 - workers: - machineDeployments: - - class: default-worker - name: md-0 - replicas: 0 From ecc2995a03d24a60ed4e1b0dda62cb163f7e6ac1 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 03:09:44 +0900 Subject: [PATCH 20/23] Delete tailscale --- .github/workflows/test-e2e.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 55e3314..9f58b79 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -16,22 +16,6 @@ jobs: - name: Clone the code uses: actions/checkout@v4 - - name: Tailscale - uses: tailscale/github-action@v2 - with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:ci - - name: Install and Run SSH server - run: | - sudo apt-get update - sudo apt-get install -y openssh-server - sudo systemctl start ssh - sudo systemctl status ssh - whoami - mkdir -p ~/.ssh - curl -L https://github.com/tsuzu.keys >> ~/.ssh/authorized_keys - - run: | cd .github && docker compose up -d docker ps -a @@ -110,10 +94,3 @@ jobs: export INCUS_TOKEN=~/.config/incus/oidctokens/incus.txt go mod tidy make test-e2e - exit 1 - - - name: Wait - if: failure() - run: | - sleep 3600 - echo "Failed" From de6a42c335d0a96290a8c0fec9fc9437dc6e6d60 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 19:47:02 +0900 Subject: [PATCH 21/23] Remove unused ctx --- internal/controller/incusmachine_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 77e55b2..0dd2766 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -277,7 +277,7 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c output, err := r.IncusClient.GetInstance(ctx, incusMachine.Name) if err == nil { - if r.isMachineReady(ctx, output) { + if r.isMachineReady(output) { log.Info("IncusMachine instance is ready") incusMachine.Spec.ProviderID = &output.ProviderID @@ -345,7 +345,7 @@ func (r *IncusMachineReconciler) getBootstrapData(ctx context.Context, namespace return string(value), bootstrapv1.Format(format), nil } -func (r *IncusMachineReconciler) isMachineReady(ctx context.Context, output *incus.GetInstanceOutput) bool { +func (r *IncusMachineReconciler) isMachineReady(output *incus.GetInstanceOutput) bool { return output.StatusCode == api.Running } From 41fad1168739a1891e3f710a19baed2d70f6c468 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Thu, 2 Jan 2025 19:55:23 +0900 Subject: [PATCH 22/23] Fix namespace for e2e --- test/e2e/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 1e63de4..9bdb4b3 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -31,7 +31,7 @@ import ( ) // namespace where the project is deployed in -const namespace = "cluster-api-provider-incus-system" +const namespace = "capincus-system" // serviceAccountName created for the project const serviceAccountName = "cluster-api-provider-incus-controller-manager" From fdc16eadb970330a88a32fd100013176338c3150 Mon Sep 17 00:00:00 2001 From: Tsuzu <8574909+tsuzu@users.noreply.github.com> Date: Fri, 3 Jan 2025 02:44:30 +0900 Subject: [PATCH 23/23] Requeue after instance creation --- internal/controller/incusmachine_controller.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/controller/incusmachine_controller.go b/internal/controller/incusmachine_controller.go index 0dd2766..8a1e594 100644 --- a/internal/controller/incusmachine_controller.go +++ b/internal/controller/incusmachine_controller.go @@ -322,7 +322,9 @@ func (r *IncusMachineReconciler) reconcileNormal(ctx context.Context, cluster *c return ctrl.Result{}, fmt.Errorf("failed to create instance: %w", err) } - return ctrl.Result{}, nil + return ctrl.Result{ + RequeueAfter: 10 * time.Second, + }, nil } func (r *IncusMachineReconciler) getBootstrapData(ctx context.Context, namespace string, dataSecretName string) (string, bootstrapv1.Format, error) {