From 976bef9bb2867199a159dd76afd093ceaf5a5e85 Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Fri, 13 Oct 2023 11:18:38 +0200 Subject: [PATCH 1/6] cluster template for vsphere Signed-off-by: Jirka Kremser --- CHANGELOG.md | 4 + cmd/template/cluster/flag.go | 55 +++- cmd/template/cluster/provider/capv.go | 276 ++++++++++++++++-- cmd/template/cluster/provider/common.go | 127 ++------ .../provider/templates/capv/functions.go | 52 ++++ .../cluster/provider/templates/capv/types.go | 87 ++++++ .../templates/vsphere/cluster.yaml.tmpl | 30 -- .../vsphere/kubeadm_config_template.yaml.tmpl | 35 --- .../vsphere/kubeadm_control_plane.yaml.tmpl | 106 ------- .../vsphere/machine_deployment.yaml.tmpl | 44 --- .../provider/templates/vsphere/template.go | 42 --- .../vsphere/vsphere_cluster.yaml.tmpl | 19 -- .../vsphere_machine_template.yaml.tmpl | 28 -- cmd/template/cluster/runner.go | 1 + pkg/template/app/app.go | 6 +- 15 files changed, 472 insertions(+), 440 deletions(-) create mode 100644 cmd/template/cluster/provider/templates/capv/functions.go create mode 100644 cmd/template/cluster/provider/templates/capv/types.go delete mode 100644 cmd/template/cluster/provider/templates/vsphere/cluster.yaml.tmpl delete mode 100644 cmd/template/cluster/provider/templates/vsphere/kubeadm_config_template.yaml.tmpl delete mode 100644 cmd/template/cluster/provider/templates/vsphere/kubeadm_control_plane.yaml.tmpl delete mode 100644 cmd/template/cluster/provider/templates/vsphere/machine_deployment.yaml.tmpl delete mode 100644 cmd/template/cluster/provider/templates/vsphere/template.go delete mode 100644 cmd/template/cluster/provider/templates/vsphere/vsphere_cluster.yaml.tmpl delete mode 100644 cmd/template/cluster/provider/templates/vsphere/vsphere_machine_template.yaml.tmpl diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a4c45b02..106c3d187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ and this project's packages adheres to [Semantic Versioning](http://semver.org/s - `kubectl-gs login`: listen only on localhost for callbacks +### Added + +- `cluster template` for up-to-date vsphere cluster app. + ## [2.43.0] - 2023-10-11 ### Added diff --git a/cmd/template/cluster/flag.go b/cmd/template/cluster/flag.go index b4ef39b35..55780321c 100644 --- a/cmd/template/cluster/flag.go +++ b/cmd/template/cluster/flag.go @@ -90,6 +90,22 @@ const ( flagOpenStackWorkerMachineFlavor = "worker-machine-flavor" flagOpenStackWorkerReplicas = "worker-replicas" + // VSphere only. + flagVSphereControlPlaneIP = "vsphere-control-plane-ip" + flagVSphereServiceLoadBalancerCIDR = "vsphere-service-load-balancer-cidr" + flagVSphereNetworkName = "vsphere-network-name" + flagVSphereControlPlaneDiskGiB = "vsphere-control-plane-disk-gib" + flagVSphereControlPlaneMemoryMiB = "vsphere-control-plane-memory-mib" + flagVSphereControlPlaneNumCPUs = "vsphere-control-plane-num-cpus" + flagVSphereControlPlaneReplicas = "vsphere-control-plane-replicas" + flagVSphereWorkerDiskGiB = "vsphere-worker-disk-gib" + flagVSphereWorkerMemoryMiB = "vsphere-worker-memory-mib" + flagVSphereWorkerNumCPUs = "vsphere-worker-num-cpus" + flagVSphereWorkerReplicas = "vsphere-worker-replicas" + flagVSphereResourcePool = "vsphere-resource-pool" + flagVSphereImageTemplate = "vsphere-image-template" + flagVSphereCredentialsSecretName = "vsphere-credentials-secret-name" // #nosec G101 + // Common. flagRegion = "region" flagBastionInstanceType = "bastion-instance-type" @@ -110,6 +126,10 @@ const ( flagRelease = "release" flagLabel = "label" flagServicePriority = "service-priority" + + // defaults + defaultKubernetesVersion = "v1.20.9" + defaultVSphereKubernetesVersion = "v1.24.11" ) type flag struct { @@ -137,6 +157,7 @@ type flag struct { Azure provider.AzureConfig GCP provider.GCPConfig OpenStack provider.OpenStackConfig + VSphere provider.VSphereConfig App provider.AppConfig OIDC provider.OIDC @@ -216,6 +237,22 @@ func (f *flag) Init(cmd *cobra.Command) { cmd.Flags().StringVar(&f.OpenStack.Worker.Flavor, flagOpenStackWorkerMachineFlavor, "", "Default worker node pool machine flavor (OpenStack only).") cmd.Flags().IntVar(&f.OpenStack.WorkerReplicas, flagOpenStackWorkerReplicas, 0, "Default worker node pool replicas (OpenStack only).") + // VSphere only + cmd.Flags().StringVar(&f.VSphere.ControlPlane.IP, flagVSphereControlPlaneIP, "", "Control plane IP, leave empty for auto allocation.") + cmd.Flags().StringVar(&f.VSphere.ServiceLoadBalancerCIDR, flagVSphereServiceLoadBalancerCIDR, "", "CIDR for Service LB for new cluster") + cmd.Flags().StringVar(&f.VSphere.NetworkName, flagVSphereNetworkName, "grasshopper-capv", "Network name in vcenter that should be used for the new VMs") + cmd.Flags().IntVar(&f.VSphere.ControlPlane.DiskGiB, flagVSphereControlPlaneDiskGiB, 50, "Disk size in GiB for control individual plane nodes") + cmd.Flags().IntVar(&f.VSphere.ControlPlane.MemoryMiB, flagVSphereControlPlaneMemoryMiB, 8096, "Memory size in MiB for individual control plane nodes") + cmd.Flags().IntVar(&f.VSphere.ControlPlane.NumCPUs, flagVSphereControlPlaneNumCPUs, 4, "Number of CPUs for individual control plane nodes") + cmd.Flags().IntVar(&f.VSphere.ControlPlane.Replicas, flagVSphereControlPlaneReplicas, 3, "Number of control plane replicas (use odd number)") + cmd.Flags().IntVar(&f.VSphere.Worker.DiskGiB, flagVSphereWorkerDiskGiB, 50, "Disk size in GiB for control individual worker nodes") + cmd.Flags().IntVar(&f.VSphere.Worker.MemoryMiB, flagVSphereWorkerMemoryMiB, 14144, "Memory size in MiB for individual worker plane nodes") + cmd.Flags().IntVar(&f.VSphere.Worker.NumCPUs, flagVSphereWorkerNumCPUs, 6, "Number of CPUs for individual worker plane nodes") + cmd.Flags().IntVar(&f.VSphere.Worker.Replicas, flagVSphereWorkerReplicas, 3, "Number of worker plane replicas") + cmd.Flags().StringVar(&f.VSphere.ResourcePool, flagVSphereResourcePool, "grasshopper", "What resource pool in vsphere should be used") + cmd.Flags().StringVar(&f.VSphere.ImageTemplate, flagVSphereImageTemplate, "ubuntu-2004-kube-%s", "OS images with Kubernetes that should be used for VMs. The '%s' will be replaced with correct Kubernetes version.") + cmd.Flags().StringVar(&f.VSphere.CredentialsSecretName, flagVSphereCredentialsSecretName, "vsphere-credentials", "Name of the secret in K8s that should be associated to cluster app. It should exist in the organization's namesapce and should contain the credentials for vsphere.") + // App-based clusters only. cmd.Flags().StringVar(&f.App.ClusterCatalog, flagClusterCatalog, "cluster", "Catalog for cluster app.") cmd.Flags().StringVar(&f.App.ClusterVersion, flagClusterVersion, "", "Version of cluster to be created.") @@ -283,7 +320,7 @@ func (f *flag) Init(cmd *cobra.Command) { cmd.Flags().StringSliceVar(&f.ControlPlaneAZ, flagControlPlaneAZ, nil, "Availability zone(s) to use by control plane nodes. Azure only supports one.") cmd.Flags().StringVar(&f.ControlPlaneInstanceType, flagControlPlaneInstanceType, "", "Instance type used for Control plane nodes") cmd.Flags().StringVar(&f.Description, flagDescription, "", "User-friendly description of the cluster's purpose (formerly called name).") - cmd.Flags().StringVar(&f.KubernetesVersion, flagKubernetesVersion, "v1.20.9", "Cluster Kubernetes version.") + cmd.Flags().StringVar(&f.KubernetesVersion, flagKubernetesVersion, defaultKubernetesVersion, "Cluster Kubernetes version.") cmd.Flags().StringVar(&f.Name, flagName, "", "Unique identifier of the cluster (formerly called ID).") cmd.Flags().StringVar(&f.OIDC.IssuerURL, flagOIDCIssuerURL, "", "OIDC issuer URL.") cmd.Flags().StringVar(&f.OIDC.CAFile, flagOIDCCAFile, "", "Path to CA file used to verify OIDC issuer (optional, OpenStack only).") @@ -388,6 +425,22 @@ func (f *flag) Validate() error { if len(f.ControlPlaneAZ) > 1 { return microerror.Maskf(invalidFlagError, "--%s supports one availability zone only", flagControlPlaneAZ) } + case key.ProviderVSphere: + if f.VSphere.ServiceLoadBalancerCIDR == "" { + return microerror.Maskf(invalidFlagError, "CIDR range from which the public IPs for Services of type LoadBalancer are taken (required) (--%s)", flagVSphereServiceLoadBalancerCIDR) + } + if !validateCIDR(f.VSphere.ServiceLoadBalancerCIDR) { + return microerror.Maskf(invalidFlagError, "--%s must be a valid CIDR", flagVSphereServiceLoadBalancerCIDR) + } + if f.KubernetesVersion == defaultKubernetesVersion { + f.KubernetesVersion = defaultVSphereKubernetesVersion + } + if f.VSphere.Worker.Replicas < 1 { + return microerror.Maskf(invalidFlagError, "--%s must be greater than 0", flagVSphereWorkerReplicas) + } + if f.VSphere.ControlPlane.Replicas < 1 { + return microerror.Maskf(invalidFlagError, "--%s must be greater than 0", flagVSphereControlPlaneReplicas) + } case key.ProviderOpenStack: if f.OpenStack.Cloud == "" { return microerror.Maskf(invalidFlagError, "--%s is required", flagOpenStackCloud) diff --git a/cmd/template/cluster/provider/capv.go b/cmd/template/cluster/provider/capv.go index ea10d8409..6cad57761 100644 --- a/cmd/template/cluster/provider/capv.go +++ b/cmd/template/cluster/provider/capv.go @@ -2,45 +2,263 @@ package provider import ( "context" + "fmt" "io" + "text/template" "github.com/giantswarm/k8sclient/v7/pkg/k8sclient" "github.com/giantswarm/microerror" + "sigs.k8s.io/yaml" - "github.com/giantswarm/kubectl-gs/v2/cmd/template/cluster/provider/templates/vsphere" + k8smetadata "github.com/giantswarm/k8smetadata/pkg/label" + + applicationv1alpha1 "github.com/giantswarm/apiextensions-application/api/v1alpha1" + + "github.com/giantswarm/kubectl-gs/v2/cmd/template/cluster/provider/templates/capv" "github.com/giantswarm/kubectl-gs/v2/internal/key" + templateapp "github.com/giantswarm/kubectl-gs/v2/pkg/template/app" ) -func WriteVSphereTemplate(ctx context.Context, client k8sclient.Interface, out io.Writer, config ClusterConfig) error { - data := struct { - Description string - KubernetesVersion string - Name string - Namespace string - Organization string - PodsCIDR string - ReleaseVersion string - SSHPublicKeys []string - }{ - Description: config.Description, - KubernetesVersion: "v1.20.1", - Name: config.Name, - Namespace: key.OrganizationNamespaceFromName(config.Organization), - Organization: config.Organization, - PodsCIDR: config.PodsCIDR, - ReleaseVersion: config.ReleaseVersion, - SSHPublicKeys: []string{}, - } - - var templates []templateConfig - for _, t := range vsphere.GetTemplates() { - templates = append(templates, templateConfig(t)) - } - - err := runMutation(ctx, client, data, templates, out) +const ( + DefaultAppsVsphereRepoName = "default-apps-vsphere" + ClusterVsphereRepoName = "cluster-vsphere" +) + +func WriteVSphereTemplate(ctx context.Context, client k8sclient.Interface, output io.Writer, config ClusterConfig) error { + err := templateClusterVSphere(ctx, client, output, config) if err != nil { return microerror.Mask(err) } - return nil + err = templateDefaultAppsVsphere(ctx, client, output, config) + return microerror.Mask(err) +} + +func templateClusterVSphere(ctx context.Context, k8sClient k8sclient.Interface, output io.Writer, config ClusterConfig) error { + appName := config.Name + configMapName := userConfigMapName(appName) + + var configMapYAML []byte + { + flagValues := BuildCapvClusterConfig(config) + + configData, err := capv.GenerateClusterValues(flagValues) + if err != nil { + return microerror.Mask(err) + } + + userConfigMap, err := templateapp.NewConfigMap(templateapp.UserConfig{ + Name: configMapName, + Namespace: organizationNamespace(config.Organization), + Data: configData, + }) + if err != nil { + return microerror.Mask(err) + } + + userConfigMap.Labels = map[string]string{} + userConfigMap.Labels[k8smetadata.Cluster] = config.Name + + configMapYAML, err = yaml.Marshal(userConfigMap) + if err != nil { + return microerror.Mask(err) + } + } + + var appYAML []byte + { + appVersion := config.App.ClusterVersion + if appVersion == "" { + var err error + appVersion, err = getLatestVersion(ctx, k8sClient.CtrlClient(), ClusterVsphereRepoName, config.App.ClusterCatalog) + if err != nil { + return microerror.Mask(err) + } + } + extraConfigs := []applicationv1alpha1.AppExtraConfig{ + { + Kind: "secret", + Name: "container-registries-configuration", + Namespace: "default", + Priority: 25, + }, + } + + clusterAppConfig := templateapp.Config{ + AppName: config.Name, + Catalog: config.App.ClusterCatalog, + InCluster: true, + Name: ClusterVsphereRepoName, + Namespace: organizationNamespace(config.Organization), + Version: appVersion, + UserConfigConfigMapName: configMapName, + UserConfigSecretName: config.VSphere.CredentialsSecretName, + ExtraConfigs: extraConfigs, + } + + var err error + appYAML, err = templateapp.NewAppCR(clusterAppConfig) + if err != nil { + return microerror.Mask(err) + } + } + + t := template.Must(template.New("appCR").Parse(key.AppCRTemplate)) + + err := t.Execute(output, templateapp.AppCROutput{ + AppCR: string(appYAML), + UserConfigConfigMap: string(configMapYAML), + }) + return microerror.Mask(err) +} + +func BuildCapvClusterConfig(config ClusterConfig) capv.ClusterConfig { + const className = "default" + return capv.ClusterConfig{ + BaseDomain: "test.gigantic.io", + ClusterDescription: config.Description, + Organization: config.Organization, + Cluster: &capv.Cluster{ + KubernetesVersion: config.KubernetesVersion, + EnableEncryptionProvider: false, + }, + Connectivity: &capv.Connectivity{ + Network: &capv.Network{ + AllowAllEgress: true, + ControlPlaneEndpoint: &capv.ControlPlaneEndpoint{ + Host: config.VSphere.ControlPlane.IP, + IpPoolName: "wc-cp-ips", + Port: 6443, + }, + LoadBalancers: &capv.LoadBalancers{ + CidrBlocks: []string{ + config.VSphere.ServiceLoadBalancerCIDR, + }, + }, + }, + }, + ControlPlane: &capv.ControlPlane{ + Replicas: config.VSphere.ControlPlane.Replicas, + Image: &capv.Image{ + Repository: "registry.k8s.io", + }, + MachineTemplate: getMachineTemplate(&config.VSphere.ControlPlane.VSphereMachineTemplate, &config), + }, + NodeClasses: map[string]*capv.MachineTemplate{ + className: getMachineTemplate(&config.VSphere.Worker, &config), + }, + NodePools: map[string]*capv.NodePool{ + "worker": { + Class: className, + Replicas: config.VSphere.Worker.Replicas, + }, + }, + HelmReleases: &capv.HelmReleases{ + Cilium: &capv.HelmRelease{ + Interval: "20s", + }, + Cpi: &capv.HelmRelease{ + Interval: "30s", + }, + Coredns: &capv.HelmRelease{ + Interval: "30s", + }, + }, + } +} + +func getMachineTemplate(machineTemplate *VSphereMachineTemplate, clusterConfig *ClusterConfig) *capv.MachineTemplate { + config := clusterConfig.VSphere + commonNetwork := &capv.MTNetwork{ + Devices: []*capv.MTDevice{ + { + NetworkName: config.NetworkName, + Dhcp4: true, + }, + }, + } + return &capv.MachineTemplate{ + Network: commonNetwork, + CloneMode: "linkedClone", + DiskGiB: machineTemplate.DiskGiB, + NumCPUs: machineTemplate.NumCPUs, + MemoryMiB: machineTemplate.MemoryMiB, + ResourcePool: config.ResourcePool, + Template: fmt.Sprintf(config.ImageTemplate, clusterConfig.KubernetesVersion), + } +} + +func templateDefaultAppsVsphere(ctx context.Context, k8sClient k8sclient.Interface, output io.Writer, config ClusterConfig) error { + appName := fmt.Sprintf("%s-default-apps", config.Name) + configMapName := userConfigMapName(appName) + + var configMapYAML []byte + { + flagValues := capv.DefaultAppsConfig{ + ClusterName: config.Name, + Organization: config.Organization, + } + + configData, err := capv.GenerateDefaultAppsValues(flagValues) + if err != nil { + return microerror.Mask(err) + } + + userConfigMap, err := templateapp.NewConfigMap(templateapp.UserConfig{ + Name: configMapName, + Namespace: organizationNamespace(config.Organization), + Data: configData, + }) + if err != nil { + return microerror.Mask(err) + } + + userConfigMap.Labels = map[string]string{} + userConfigMap.Labels[k8smetadata.Cluster] = config.Name + + configMapYAML, err = yaml.Marshal(userConfigMap) + if err != nil { + return microerror.Mask(err) + } + } + + var appYAML []byte + { + appVersion := config.App.DefaultAppsVersion + if appVersion == "" { + var err error + appVersion, err = getLatestVersion(ctx, k8sClient.CtrlClient(), DefaultAppsVsphereRepoName, config.App.DefaultAppsCatalog) + if err != nil { + return microerror.Mask(err) + } + } + + var err error + appYAML, err = templateapp.NewAppCR(templateapp.Config{ + AppName: appName, + Cluster: config.Name, + Catalog: config.App.DefaultAppsCatalog, + DefaultingEnabled: false, + InCluster: true, + Name: DefaultAppsVsphereRepoName, + Namespace: organizationNamespace(config.Organization), + Version: appVersion, + UserConfigConfigMapName: configMapName, + UseClusterValuesConfig: true, + ExtraLabels: map[string]string{ + k8smetadata.ManagedBy: "cluster", + }, + }) + if err != nil { + return microerror.Mask(err) + } + } + + t := template.Must(template.New("appCR").Parse(key.AppCRTemplate)) + + err := t.Execute(output, templateapp.AppCROutput{ + UserConfigConfigMap: string(configMapYAML), + AppCR: string(appYAML), + }) + return microerror.Mask(err) } diff --git a/cmd/template/cluster/provider/common.go b/cmd/template/cluster/provider/common.go index b7d864cde..02557af22 100644 --- a/cmd/template/cluster/provider/common.go +++ b/cmd/template/cluster/provider/common.go @@ -1,13 +1,10 @@ package provider import ( - "bytes" "context" "fmt" - "io" "math" "strings" - "text/template" applicationv1alpha1 "github.com/giantswarm/apiextensions-application/api/v1alpha1" "github.com/giantswarm/k8sclient/v7/pkg/k8sclient" @@ -17,16 +14,9 @@ import ( "github.com/giantswarm/micrologger" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" - apiyaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - "k8s.io/client-go/discovery" - memory "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/restmapper" capi "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" "github.com/giantswarm/kubectl-gs/v2/pkg/app" ) @@ -72,6 +62,28 @@ type GCPConfig struct { MachineDeployment GCPMachineDeployment } +type VSphereConfig struct { + ControlPlane VSphereControlPlane + CredentialsSecretName string + ImageTemplate string + NetworkName string + Worker VSphereMachineTemplate + ResourcePool string + ServiceLoadBalancerCIDR string +} + +type VSphereMachineTemplate struct { + DiskGiB int + MemoryMiB int + NumCPUs int + Replicas int +} + +type VSphereControlPlane struct { + IP string + VSphereMachineTemplate +} + type GCPControlPlane struct { ServiceAccount ServiceAccount } @@ -146,6 +158,7 @@ type ClusterConfig struct { App AppConfig AWS AWSConfig Azure AzureConfig + VSphere VSphereConfig GCP GCPConfig OpenStack OpenStackConfig } @@ -158,11 +171,6 @@ type OIDC struct { GroupsClaim string } -type templateConfig struct { - Name string - Data string -} - func newcapiClusterCR(config ClusterConfig, infrastructureRef *corev1.ObjectReference) *capi.Cluster { cluster := &capi.Cluster{ TypeMeta: metav1.TypeMeta{ @@ -200,95 +208,6 @@ func newcapiClusterCR(config ClusterConfig, infrastructureRef *corev1.ObjectRefe return cluster } -func runMutation(ctx context.Context, client k8sclient.Interface, templateData interface{}, templates []templateConfig, output io.Writer) error { - var err error - - // Create a DiscoveryClient to get the GVR. - dc, err := discovery.NewDiscoveryClientForConfig(client.RESTConfig()) - if err != nil { - return microerror.Mask(err) - } - - // Create a DynamicClient to execute our request. - dyn, err := dynamic.NewForConfig(client.RESTConfig()) - if err != nil { - return microerror.Mask(err) - } - - mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) - for _, t := range templates { - // Add separators to make the entire file valid yaml and allow easy appending. - _, err = output.Write([]byte("---\n")) - if err != nil { - return microerror.Mask(err) - } - te := template.Must(template.New(t.Name).Parse(t.Data)) - var buf bytes.Buffer - // Template from our inputs. - err = te.Execute(&buf, templateData) - if err != nil { - return microerror.Mask(err) - } - - // Transform to unstructured.Unstructured. - obj := &unstructured.Unstructured{} - dec := apiyaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) - _, gvk, err := dec.Decode(buf.Bytes(), nil, obj) - if err != nil { - return microerror.Mask(err) - } - // Mapping needed for the dynamic client. - mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return microerror.Mask(err) - } - // Get namespace information from object. - namespace, namespaced, err := unstructured.NestedString(obj.Object, "metadata", "namespace") - if err != nil { - return microerror.Mask(err) - } - var defaultedObj *unstructured.Unstructured - // Execute our request as a `DryRun` - no resources will be persisted. - if namespaced { - defaultedObj, err = dyn.Resource(mapping.Resource).Namespace(namespace).Create(ctx, obj, metav1.CreateOptions{DryRun: []string{"All"}}) - if err != nil { - // TODO handle different kinds of errors (e.g. validation) here. - return microerror.Mask(err) - } - } else { - defaultedObj, err = dyn.Resource(mapping.Resource).Create(ctx, obj, metav1.CreateOptions{DryRun: []string{"All"}}) - if err != nil { - // TODO handle different kinds of errors (e.g. validation) here. - return microerror.Mask(err) - } - } - // Strip `managedFields` for better readability. - unstructured.RemoveNestedField(defaultedObj.Object, "metadata", "managedFields") - // Strip some metadata fields for better UX. - unstructured.RemoveNestedField(defaultedObj.Object, "metadata", "creationTimestamp") - unstructured.RemoveNestedField(defaultedObj.Object, "metadata", "generation") - unstructured.RemoveNestedField(defaultedObj.Object, "metadata", "selfLink") - unstructured.RemoveNestedField(defaultedObj.Object, "metadata", "uid") - // Unstructured to JSON. - buf = *new(bytes.Buffer) - err = dec.Encode(defaultedObj, &buf) - if err != nil { - return microerror.Mask(err) - } - // JSON to YAML. - mutated, err := yaml.JSONToYAML(buf.Bytes()) - if err != nil { - return microerror.Mask(err) - } - // Write the yaml to our file. - _, err = output.Write(mutated) - if err != nil { - return microerror.Mask(err) - } - } - return nil -} - func getLatestVersion(ctx context.Context, ctrlClient client.Client, app, catalog string) (string, error) { var catalogEntryList applicationv1alpha1.AppCatalogEntryList err := ctrlClient.List(ctx, &catalogEntryList, &client.ListOptions{ diff --git a/cmd/template/cluster/provider/templates/capv/functions.go b/cmd/template/cluster/provider/templates/capv/functions.go new file mode 100644 index 000000000..fe6b8105a --- /dev/null +++ b/cmd/template/cluster/provider/templates/capv/functions.go @@ -0,0 +1,52 @@ +package capv + +import ( + "github.com/giantswarm/microerror" + "sigs.k8s.io/yaml" +) + +func GenerateClusterValues(flagInputs ClusterConfig) (string, error) { + var flagConfigData map[string]interface{} + + { + flagConfigYAML, err := yaml.Marshal(flagInputs) + if err != nil { + return "", microerror.Mask(err) + } + + err = yaml.Unmarshal(flagConfigYAML, &flagConfigData) + if err != nil { + return "", microerror.Mask(err) + } + } + + finalConfigString, err := yaml.Marshal(flagInputs) + if err != nil { + return "", microerror.Mask(err) + } + + return string(finalConfigString), nil +} + +func GenerateDefaultAppsValues(flagConfig DefaultAppsConfig) (string, error) { + var flagConfigData map[string]interface{} + + { + flagConfigYAML, err := yaml.Marshal(flagConfig) + if err != nil { + return "", microerror.Mask(err) + } + + err = yaml.Unmarshal(flagConfigYAML, &flagConfigData) + if err != nil { + return "", microerror.Mask(err) + } + } + + finalConfigString, err := yaml.Marshal(flagConfigData) + if err != nil { + return "", microerror.Mask(err) + } + + return string(finalConfigString), nil +} diff --git a/cmd/template/cluster/provider/templates/capv/types.go b/cmd/template/cluster/provider/templates/capv/types.go new file mode 100644 index 000000000..f26b74dac --- /dev/null +++ b/cmd/template/cluster/provider/templates/capv/types.go @@ -0,0 +1,87 @@ +package capv + +type ClusterConfig struct { + Connectivity *Connectivity `json:"connectivity,omitempty"` + ControlPlane *ControlPlane `json:"controlPlane,omitempty"` + NodeClasses map[string]*MachineTemplate `json:"nodeClasses,omitempty"` + NodePools map[string]*NodePool `json:"nodePools,omitempty"` + HelmReleases *HelmReleases `json:"helmReleases,omitempty"` + BaseDomain string `json:"baseDomain,omitempty"` + ClusterDescription string `json:"clusterDescription,omitempty"` + Organization string `json:"organization,omitempty"` + Cluster *Cluster `json:"cluster,omitempty"` +} + +type Cluster struct { + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + EnableEncryptionProvider bool `json:"enableEncryptionProvider,omitempty"` +} + +type DefaultAppsConfig struct { + ClusterName string `json:"clusterName,omitempty"` + Organization string `json:"organization,omitempty"` +} + +type Connectivity struct { + Network *Network `json:"network,omitempty"` +} + +type Network struct { + AllowAllEgress bool `json:"allowAllEgress,omitempty"` + ControlPlaneEndpoint *ControlPlaneEndpoint `json:"controlPlaneEndpoint,omitempty"` + LoadBalancers *LoadBalancers `json:"loadBalancers,omitempty"` +} + +type ControlPlaneEndpoint struct { + Host string `json:"host"` + Port int `json:"port,omitempty"` + IpPoolName string `json:"ipPoolName,omitempty"` +} + +type LoadBalancers struct { + CidrBlocks []string `json:"cidrBlocks,omitempty"` +} + +type ControlPlane struct { + Image *Image `json:"image,omitempty"` + Replicas int `json:"replicas,omitempty"` + MachineTemplate *MachineTemplate `json:"machineTemplate,omitempty"` +} + +type MTNetwork struct { + Devices []*MTDevice `json:"devices,omitempty"` +} + +type MTDevice struct { + NetworkName string `json:"networkName,omitempty"` + Dhcp4 bool `json:"dhcp4,omitempty"` +} + +type Image struct { + Repository string `json:"repository,omitempty"` +} + +type MachineTemplate struct { + Network *MTNetwork `json:"network,omitempty"` + CloneMode string `json:"cloneMode,omitempty"` + DiskGiB int `json:"diskGiB,omitempty"` + NumCPUs int `json:"numCPUs,omitempty"` + MemoryMiB int `json:"memoryMiB,omitempty"` + ResourcePool string `json:"resourcePool,omitempty"` + Template string `json:"template,omitempty"` +} + +type NodePool struct { + Class string `json:"class,omitempty"` + Replicas int `json:"replicas,omitempty"` +} + +type HelmReleases struct { + Cilium *HelmRelease `json:"cilium,omitempty"` + Cpi *HelmRelease `json:"cpi,omitempty"` + Coredns *HelmRelease `json:"coredns,omitempty"` +} + +type HelmRelease struct { + Interval string `json:"interval,omitempty"` +} diff --git a/cmd/template/cluster/provider/templates/vsphere/cluster.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/cluster.yaml.tmpl deleted file mode 100644 index 7b5be8e02..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/cluster.yaml.tmpl +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: Cluster -metadata: - annotations: - cluster.giantswarm.io/description: {{ .Description }} - labels: - release.giantswarm.io/version: {{ .ReleaseVersion }} - giantswarm.io/cluster: {{ .Name }} - cluster.x-k8s.io/cluster-name: {{ .Name }} - giantswarm.io/organization: {{ .Organization }} - name: {{ .Name }} - namespace: {{ .Namespace }} -spec: -{{- if .PodsCIDR }} - clusterNetwork: - pods: - cidrBlocks: - - {{ .PodsCIDR }} -{{- end }} - controlPlaneEndpoint: - host: 10.0.6.191 - port: 6443 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 - kind: KubeadmControlPlane - name: {{ .Name }} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: VSphereCluster - name: {{ .Name }} diff --git a/cmd/template/cluster/provider/templates/vsphere/kubeadm_config_template.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/kubeadm_config_template.yaml.tmpl deleted file mode 100644 index 878b879fa..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/kubeadm_config_template.yaml.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 -kind: KubeadmConfigTemplate -metadata: - metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ .Name }} - giantswarm.io/cluster: {{ .Name }} - giantswarm.io/organization: {{ .Organization }} - release.giantswarm.io/version: {{ .ReleaseVersion }} - name: {{ .Name }}-md-0 - namespace: {{ .Namespace }} -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: '{{ `{{ ds.meta_data.hostname }}` }}' - preKubeadmCommands: - - hostname "{{ `{{ ds.meta_data.hostname }}` }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ `{{ ds.meta_data.hostname }}` }}" >>/etc/hosts - - echo "{{ `{{ ds.meta_data.hostname }}` }}" >/etc/hostname - users: - - name: capv - {{ with .SSHPublicKeys }} - sshAuthorizedKeys: - {{ range . }} - - "{{ . }}" - {{ end }} - {{ end }} - sudo: ALL=(ALL) NOPASSWD:ALL diff --git a/cmd/template/cluster/provider/templates/vsphere/kubeadm_control_plane.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/kubeadm_control_plane.yaml.tmpl deleted file mode 100644 index 01e1f0b74..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/kubeadm_control_plane.yaml.tmpl +++ /dev/null @@ -1,106 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 -kind: KubeadmControlPlane -metadata: - labels: - "release.giantswarm.io/version": "{{ .ReleaseVersion }}" - "giantswarm.io/cluster": "{{ .Name }}" - "cluster.x-k8s.io/cluster-name": "{{ .Name }}" - "giantswarm.io/organization": "{{ .Organization }}" - name: {{ .Name }} - namespace: {{ .Namespace }} -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - start - env: - - name: vip_arp - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_address - value: 10.0.6.191 - - name: vip_interface - value: eth0 - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - image: plndr/kube-vip:0.3.2 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - SYS_TIME - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - initConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: '{{ `{{ ds.meta_data.hostname }}` }}' - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: '{{ `{{ ds.meta_data.hostname }}` }}' - preKubeadmCommands: - - hostname "{{ `{{ ds.meta_data.hostname }}` }}" - - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - - echo "127.0.0.1 localhost" >>/etc/hosts - - echo "127.0.0.1 {{ `{{ ds.meta_data.hostname }}` }}" >>/etc/hosts - - echo "{{ `{{ ds.meta_data.hostname }}` }}" >/etc/hostname - useExperimentalRetryJoin: true - users: - - name: capv - {{ with .SSHPublicKeys }} - sshAuthorizedKeys: - {{ range . }} - - "{{ . }}" - {{ end }} - {{ end }} - sudo: ALL=(ALL) NOPASSWD:ALL - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: VSphereMachineTemplate - name: {{ .Name }} - replicas: 1 - rolloutStrategy: - rollingUpdate: - maxSurge: 1 - type: RollingUpdate - version: {{ .KubernetesVersion }} diff --git a/cmd/template/cluster/provider/templates/vsphere/machine_deployment.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/machine_deployment.yaml.tmpl deleted file mode 100644 index 6f97ac27d..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/machine_deployment.yaml.tmpl +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: MachineDeployment -metadata: - metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ .Name }} - cluster.x-k8s.io/watch-filter: capi - giantswarm.io/cluster: {{ .Name }} - giantswarm.io/organization: {{ .Organization }} - release.giantswarm.io/version: {{ .ReleaseVersion }} - name: {{ .Name }}-md - namespace: {{ .Namespace }} -spec: - clusterName: {{ .Name }} - minReadySeconds: 0 - progressDeadlineSeconds: 600 - replicas: 3 - revisionHistoryLimit: 1 - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: {{ .Name }} - cluster.x-k8s.io/deployment-name: {{ .Name }}-md-0 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - cluster.x-k8s.io/cluster-name: {{ .Name }} - cluster.x-k8s.io/deployment-name: {{ .Name }}-md-0 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 - kind: KubeadmConfigTemplate - name: {{ .Name }}-md-0 - clusterName: {{ .Name }} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: VSphereMachineTemplate - name: {{ .Name }} - version: v1.20.1 diff --git a/cmd/template/cluster/provider/templates/vsphere/template.go b/cmd/template/cluster/provider/templates/vsphere/template.go deleted file mode 100644 index b68247bdb..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/template.go +++ /dev/null @@ -1,42 +0,0 @@ -package vsphere - -import ( - _ "embed" -) - -//go:embed cluster.yaml.tmpl -var cluster string - -//go:embed vsphere_cluster.yaml.tmpl -var vsphereCluster string - -//go:embed kubeadm_control_plane.yaml.tmpl -var kubeadmControlPlane string - -//go:embed kubeadm_config_template.yaml.tmpl -var kubeadmConfigTemplate string - -//go:embed machine_deployment.yaml.tmpl -var machineDeployment string - -//go:embed vsphere_machine_template.yaml.tmpl -var vsphereMachineTemplate string - -type Template struct { - Name string - Data string -} - -// GetTemplate merges all .tmpl files. -func GetTemplates() []Template { - // Order is important here. - // The order in this slice determines in which order files will be applied. - return []Template{ - {Name: "cluster.yaml.tmpl", Data: cluster}, - {Name: "vsphere_cluster.yaml.tmpl", Data: vsphereCluster}, - {Name: "kubeadm_control_plane.yaml.tmpl", Data: kubeadmControlPlane}, - {Name: "kubeadm_config_template.yaml.tmpl", Data: kubeadmConfigTemplate}, - {Name: "machine_deployment.yaml.tmpl", Data: machineDeployment}, - {Name: "vsphere_machine_template.yaml.tmpl", Data: vsphereMachineTemplate}, - } -} diff --git a/cmd/template/cluster/provider/templates/vsphere/vsphere_cluster.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/vsphere_cluster.yaml.tmpl deleted file mode 100644 index 3d387ebfa..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/vsphere_cluster.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: VSphereCluster -metadata: - labels: - "release.giantswarm.io/version": "{{ .ReleaseVersion }}" - "giantswarm.io/cluster": "{{ .Name }}" - "cluster.x-k8s.io/cluster-name": "{{ .Name }}" - "giantswarm.io/organization": "{{ .Organization }}" - name: {{ .Name }} - namespace: {{ .Namespace }} -spec: - controlPlaneEndpoint: - host: 10.0.6.191 #CHANGE HERE - port: 6443 - identityRef: - kind: Secret - name: {{ .Name }} - server: vcenter-rhr3c72bx1.ionoscloud.tools - thumbprint: 4E:79:A1:50:3E:11:C6:CB:34:7A:A8:C8:93:1A:3D:AA:96:AF:73:03 diff --git a/cmd/template/cluster/provider/templates/vsphere/vsphere_machine_template.yaml.tmpl b/cmd/template/cluster/provider/templates/vsphere/vsphere_machine_template.yaml.tmpl deleted file mode 100644 index 9221b6bc2..000000000 --- a/cmd/template/cluster/provider/templates/vsphere/vsphere_machine_template.yaml.tmpl +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: VSphereMachineTemplate -metadata: - labels: - "release.giantswarm.io/version": "{{ .ReleaseVersion }}" - "giantswarm.io/cluster": "{{ .Name }}" - "cluster.x-k8s.io/cluster-name": "{{ .Name }}" - "giantswarm.io/organization": "{{ .Organization }}" - name: {{ .Name }} - namespace: {{ .Namespace }} -spec: - template: - spec: - cloneMode: linkedClone - datacenter: Datacenter - datastore: vsanDatastore - diskGiB: 25 - folder: capi - memoryMiB: 8192 - network: - devices: - - dhcp4: true - networkName: gopher-management-cluster - numCPUs: 2 - resourcePool: Cluster1/Resources - server: vcenter-rhr3c72bx1.ionoscloud.tools - template: ubuntu-1804-kube-v1.20.1 - thumbprint: 4E:79:A1:50:3E:11:C6:CB:34:7A:A8:C8:93:1A:3D:AA:96:AF:73:03 diff --git a/cmd/template/cluster/runner.go b/cmd/template/cluster/runner.go index 717043bbc..fc917ea46 100644 --- a/cmd/template/cluster/runner.go +++ b/cmd/template/cluster/runner.go @@ -137,6 +137,7 @@ func (r *runner) getClusterConfig() (provider.ClusterConfig, error) { GCP: r.flag.GCP, OIDC: r.flag.OIDC, OpenStack: r.flag.OpenStack, + VSphere: r.flag.VSphere, } if config.Name == "" { diff --git a/pkg/template/app/app.go b/pkg/template/app/app.go index aa63935c1..b2c5a332d 100644 --- a/pkg/template/app/app.go +++ b/pkg/template/app/app.go @@ -31,6 +31,7 @@ type Config struct { UpgradeTimeout *metav1.Duration UserConfigConfigMapName string UserConfigSecretName string + ExtraConfigs []applicationv1alpha1.AppExtraConfig Organization string RollbackTimeout *metav1.Duration Version string @@ -114,8 +115,9 @@ func NewAppCR(config Config) ([]byte, error) { KubeConfig: applicationv1alpha1.AppSpecKubeConfig{ InCluster: config.InCluster, }, - UserConfig: userConfig, - Version: config.Version, + UserConfig: userConfig, + ExtraConfigs: config.ExtraConfigs, + Version: config.Version, NamespaceConfig: applicationv1alpha1.AppSpecNamespaceConfig{ Annotations: config.NamespaceConfigAnnotations, Labels: config.NamespaceConfigLabels, From 367df1d06a5c3afed41c6fb132a612ff5a6ab6a2 Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Fri, 13 Oct 2023 12:48:14 +0200 Subject: [PATCH 2/6] CVE-2023-39325 > nancy-ignore Signed-off-by: Jirka Kremser --- .nancy-ignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.nancy-ignore b/.nancy-ignore index 1596db0e1..40daf7816 100644 --- a/.nancy-ignore +++ b/.nancy-ignore @@ -5,3 +5,6 @@ CVE-2022-29153 until=2023-11-01 # # This is present in the current latest v0.26.0 as well CVE-2020-8561 until=2023-11-01 +# pkg:golang/golang.org/x/net@v0.13.0 the security fix hasn't been backported to golang@1.19 yet - https://github.com/golang/go/issues/63417 +CVE-2023-39325 until=2023-12-01 + From 7a3669a25f1b362f50082e6b1e03b1a3a63b821e Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Mon, 23 Oct 2023 13:22:39 +0200 Subject: [PATCH 3/6] add unit test Signed-off-by: Jirka Kremser --- cmd/template/cluster/runner_test.go | 39 +++++ .../testdata/run_template_cluster_capv.golden | 149 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 cmd/template/cluster/testdata/run_template_cluster_capv.golden diff --git a/cmd/template/cluster/runner_test.go b/cmd/template/cluster/runner_test.go index 1a95851f0..f452b738c 100644 --- a/cmd/template/cluster/runner_test.go +++ b/cmd/template/cluster/runner_test.go @@ -201,6 +201,45 @@ func Test_run(t *testing.T) { args: nil, expectedGoldenFile: "run_template_cluster_capz.golden", }, + { + name: "case 5: template cluster capv", + flags: &flag{ + Name: "test1", + Provider: "vsphere", + Description: "yet another test cluster", + Organization: "test", + KubernetesVersion: "v1.2.3", + App: provider.AppConfig{ + ClusterVersion: "1.2.3", + ClusterCatalog: "foo-catalog", + DefaultAppsCatalog: "foo-default-catalog", + DefaultAppsVersion: "3.2.1", + }, + VSphere: provider.VSphereConfig{ + ServiceLoadBalancerCIDR: "1.2.3.4/32", + ResourcePool: "foopool", + NetworkName: "foonet", + CredentialsSecretName: "foosecret", + ImageTemplate: "foo-%-os", + ControlPlane: provider.VSphereControlPlane{ + VSphereMachineTemplate: provider.VSphereMachineTemplate{ + DiskGiB: 42, + MemoryMiB: 42000, + NumCPUs: 6, + Replicas: 5, + }, + }, + Worker: provider.VSphereMachineTemplate{ + DiskGiB: 43, + MemoryMiB: 43000, + NumCPUs: 7, + Replicas: 4, + }, + }, + }, + args: nil, + expectedGoldenFile: "run_template_cluster_capv.golden", + }, } for _, tc := range testCases { diff --git a/cmd/template/cluster/testdata/run_template_cluster_capv.golden b/cmd/template/cluster/testdata/run_template_cluster_capv.golden new file mode 100644 index 000000000..4abdd61fb --- /dev/null +++ b/cmd/template/cluster/testdata/run_template_cluster_capv.golden @@ -0,0 +1,149 @@ +--- +apiVersion: v1 +data: + values: | + baseDomain: test.gigantic.io + cluster: + kubernetesVersion: v1.2.3 + clusterDescription: yet another test cluster + connectivity: + network: + allowAllEgress: true + controlPlaneEndpoint: + host: "" + ipPoolName: wc-cp-ips + port: 6443 + loadBalancers: + cidrBlocks: + - 1.2.3.4/32 + controlPlane: + image: + repository: registry.k8s.io + machineTemplate: + cloneMode: linkedClone + diskGiB: 42 + memoryMiB: 42000 + network: + devices: + - dhcp4: true + networkName: foonet + numCPUs: 6 + resourcePool: foopool + template: foo-%!o(string=v1.2.3)s + replicas: 5 + helmReleases: + cilium: + interval: 20s + coredns: + interval: 30s + cpi: + interval: 30s + nodeClasses: + default: + cloneMode: linkedClone + diskGiB: 43 + memoryMiB: 43000 + network: + devices: + - dhcp4: true + networkName: foonet + numCPUs: 7 + resourcePool: foopool + template: foo-%!o(string=v1.2.3)s + nodePools: + worker: + class: default + replicas: 4 + organization: test +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + giantswarm.io/cluster: test1 + name: test1-userconfig + namespace: org-test +--- +apiVersion: application.giantswarm.io/v1alpha1 +kind: App +metadata: + labels: + app-operator.giantswarm.io/version: 0.0.0 + name: test1 + namespace: org-test +spec: + catalog: foo-catalog + config: + configMap: + name: "" + namespace: "" + secret: + name: "" + namespace: "" + extraConfigs: + - kind: secret + name: container-registries-configuration + namespace: default + priority: 25 + kubeConfig: + context: + name: "" + inCluster: true + secret: + name: "" + namespace: "" + name: cluster-vsphere + namespace: org-test + userConfig: + configMap: + name: test1-userconfig + namespace: org-test + secret: + name: foosecret + namespace: org-test + version: 1.2.3 +--- +apiVersion: v1 +data: + values: | + clusterName: test1 + organization: test +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + giantswarm.io/cluster: test1 + name: test1-default-apps-userconfig + namespace: org-test +--- +apiVersion: application.giantswarm.io/v1alpha1 +kind: App +metadata: + labels: + app-operator.giantswarm.io/version: 0.0.0 + giantswarm.io/cluster: test1 + giantswarm.io/managed-by: cluster + name: test1-default-apps + namespace: org-test +spec: + catalog: foo-default-catalog + config: + configMap: + name: test1-cluster-values + namespace: org-test + secret: + name: "" + namespace: "" + kubeConfig: + context: + name: "" + inCluster: true + secret: + name: "" + namespace: "" + name: default-apps-vsphere + namespace: org-test + userConfig: + configMap: + name: test1-default-apps-userconfig + namespace: org-test + version: 3.2.1 From 651bf8ee724337629d63f26b5c0b3cb12da30b5e Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Mon, 23 Oct 2023 18:13:30 +0200 Subject: [PATCH 4/6] capv: add configurable ip pool name Signed-off-by: Jirka Kremser --- cmd/template/cluster/flag.go | 2 ++ cmd/template/cluster/provider/capv.go | 2 +- cmd/template/cluster/provider/common.go | 3 ++- cmd/template/cluster/runner_test.go | 1 + cmd/template/cluster/testdata/run_template_cluster_capv.golden | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/template/cluster/flag.go b/cmd/template/cluster/flag.go index 55780321c..05a882f51 100644 --- a/cmd/template/cluster/flag.go +++ b/cmd/template/cluster/flag.go @@ -95,6 +95,7 @@ const ( flagVSphereServiceLoadBalancerCIDR = "vsphere-service-load-balancer-cidr" flagVSphereNetworkName = "vsphere-network-name" flagVSphereControlPlaneDiskGiB = "vsphere-control-plane-disk-gib" + flagVSphereControlPlaneIpPool = "vsphere-control-plane-ip-pool" flagVSphereControlPlaneMemoryMiB = "vsphere-control-plane-memory-mib" flagVSphereControlPlaneNumCPUs = "vsphere-control-plane-num-cpus" flagVSphereControlPlaneReplicas = "vsphere-control-plane-replicas" @@ -241,6 +242,7 @@ func (f *flag) Init(cmd *cobra.Command) { cmd.Flags().StringVar(&f.VSphere.ControlPlane.IP, flagVSphereControlPlaneIP, "", "Control plane IP, leave empty for auto allocation.") cmd.Flags().StringVar(&f.VSphere.ServiceLoadBalancerCIDR, flagVSphereServiceLoadBalancerCIDR, "", "CIDR for Service LB for new cluster") cmd.Flags().StringVar(&f.VSphere.NetworkName, flagVSphereNetworkName, "grasshopper-capv", "Network name in vcenter that should be used for the new VMs") + cmd.Flags().StringVar(&f.VSphere.ControlPlane.IPPoolName, flagVSphereControlPlaneIpPool, "wc-cp-ips", "Name of `GlobalInClusterIpPool` CR from which the IP for CP is taken") cmd.Flags().IntVar(&f.VSphere.ControlPlane.DiskGiB, flagVSphereControlPlaneDiskGiB, 50, "Disk size in GiB for control individual plane nodes") cmd.Flags().IntVar(&f.VSphere.ControlPlane.MemoryMiB, flagVSphereControlPlaneMemoryMiB, 8096, "Memory size in MiB for individual control plane nodes") cmd.Flags().IntVar(&f.VSphere.ControlPlane.NumCPUs, flagVSphereControlPlaneNumCPUs, 4, "Number of CPUs for individual control plane nodes") diff --git a/cmd/template/cluster/provider/capv.go b/cmd/template/cluster/provider/capv.go index 6cad57761..52ddde098 100644 --- a/cmd/template/cluster/provider/capv.go +++ b/cmd/template/cluster/provider/capv.go @@ -127,7 +127,7 @@ func BuildCapvClusterConfig(config ClusterConfig) capv.ClusterConfig { AllowAllEgress: true, ControlPlaneEndpoint: &capv.ControlPlaneEndpoint{ Host: config.VSphere.ControlPlane.IP, - IpPoolName: "wc-cp-ips", + IpPoolName: config.VSphere.ControlPlane.IPPoolName, Port: 6443, }, LoadBalancers: &capv.LoadBalancers{ diff --git a/cmd/template/cluster/provider/common.go b/cmd/template/cluster/provider/common.go index 02557af22..afab894b3 100644 --- a/cmd/template/cluster/provider/common.go +++ b/cmd/template/cluster/provider/common.go @@ -80,7 +80,8 @@ type VSphereMachineTemplate struct { } type VSphereControlPlane struct { - IP string + IP string + IPPoolName string VSphereMachineTemplate } diff --git a/cmd/template/cluster/runner_test.go b/cmd/template/cluster/runner_test.go index f452b738c..fa1ce8c45 100644 --- a/cmd/template/cluster/runner_test.go +++ b/cmd/template/cluster/runner_test.go @@ -228,6 +228,7 @@ func Test_run(t *testing.T) { NumCPUs: 6, Replicas: 5, }, + IPPoolName: "foo-pool", }, Worker: provider.VSphereMachineTemplate{ DiskGiB: 43, diff --git a/cmd/template/cluster/testdata/run_template_cluster_capv.golden b/cmd/template/cluster/testdata/run_template_cluster_capv.golden index 4abdd61fb..bb89ea267 100644 --- a/cmd/template/cluster/testdata/run_template_cluster_capv.golden +++ b/cmd/template/cluster/testdata/run_template_cluster_capv.golden @@ -11,7 +11,7 @@ data: allowAllEgress: true controlPlaneEndpoint: host: "" - ipPoolName: wc-cp-ips + ipPoolName: foo-pool port: 6443 loadBalancers: cidrBlocks: From 92d53796b3cd1e9f299b95831c7d841d55bb4e1e Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Mon, 23 Oct 2023 21:19:17 +0200 Subject: [PATCH 5/6] capv: Better logic for default k8s version Signed-off-by: Jirka Kremser --- cmd/template/cluster/flag.go | 5 +++-- cmd/template/cluster/runner.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/template/cluster/flag.go b/cmd/template/cluster/flag.go index 05a882f51..d066f8796 100644 --- a/cmd/template/cluster/flag.go +++ b/cmd/template/cluster/flag.go @@ -351,7 +351,7 @@ func (f *flag) Init(cmd *cobra.Command) { f.print.AddFlags(cmd) } -func (f *flag) Validate() error { +func (f *flag) Validate(cmd *cobra.Command) error { var err error validProviders := []string{ key.ProviderAWS, @@ -434,7 +434,8 @@ func (f *flag) Validate() error { if !validateCIDR(f.VSphere.ServiceLoadBalancerCIDR) { return microerror.Maskf(invalidFlagError, "--%s must be a valid CIDR", flagVSphereServiceLoadBalancerCIDR) } - if f.KubernetesVersion == defaultKubernetesVersion { + ver, err := cmd.Flags().GetString(flagKubernetesVersion) + if err != nil || ver == "" { f.KubernetesVersion = defaultVSphereKubernetesVersion } if f.VSphere.Worker.Replicas < 1 { diff --git a/cmd/template/cluster/runner.go b/cmd/template/cluster/runner.go index fc917ea46..e1861c971 100644 --- a/cmd/template/cluster/runner.go +++ b/cmd/template/cluster/runner.go @@ -35,7 +35,7 @@ func (r *runner) Run(cmd *cobra.Command, args []string) error { return r.flag.ControlPlaneAZ[i] < r.flag.ControlPlaneAZ[j] }) - err := r.flag.Validate() + err := r.flag.Validate(cmd) if err != nil { return microerror.Mask(err) } From 176473973d388187c555dd3a420dea925e8b0a06 Mon Sep 17 00:00:00 2001 From: Jirka Kremser Date: Tue, 24 Oct 2023 14:01:34 +0200 Subject: [PATCH 6/6] capv: add support also for 'k gs gitops add base --provider vsphere' Signed-off-by: Jirka Kremser --- cmd/gitops/add/base/flag.go | 1 + cmd/gitops/add/base/runner.go | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cmd/gitops/add/base/flag.go b/cmd/gitops/add/base/flag.go index 73dfe04b3..d7812803c 100644 --- a/cmd/gitops/add/base/flag.go +++ b/cmd/gitops/add/base/flag.go @@ -28,6 +28,7 @@ func supportedProviders() []string { key.ProviderCAPZ, key.ProviderGCP, key.ProviderOpenStack, + key.ProviderVSphere, } } diff --git a/cmd/gitops/add/base/runner.go b/cmd/gitops/add/base/runner.go index 869f961cf..2e94a8029 100644 --- a/cmd/gitops/add/base/runner.go +++ b/cmd/gitops/add/base/runner.go @@ -9,6 +9,7 @@ import ( templateapp "github.com/giantswarm/kubectl-gs/v2/pkg/template/app" + "github.com/giantswarm/kubectl-gs/v2/cmd/template/cluster/provider/templates/capv" "github.com/giantswarm/kubectl-gs/v2/cmd/template/cluster/provider/templates/capz" "github.com/giantswarm/kubectl-gs/v2/cmd/template/cluster/provider/templates/openstack" "github.com/giantswarm/kubectl-gs/v2/internal/gitops/filesystem/creator" @@ -102,6 +103,8 @@ func generateClusterBaseTemplates(config common.StructureConfig) (common.Cluster return generateCapGClusterBaseTemplates(config) case key.ProviderOpenStack: return generateCapOClusterBaseTemplates(config) + case key.ProviderVSphere: + return generateCapVClusterBaseTemplates(config) } return common.ClusterBaseTemplates{}, invalidProviderError @@ -285,6 +288,48 @@ func generateCapOClusterBaseTemplates(structureConfig common.StructureConfig) (c return clusterBaseTemplates, nil } +func generateCapVClusterBaseTemplates(structureConfig common.StructureConfig) (common.ClusterBaseTemplates, error) { + clusterBaseTemplates := common.ClusterBaseTemplates{} + + clusterAppCr, err := generateClusterAppCrTemplate("cluster-vsphere") + + if err != nil { + return clusterBaseTemplates, err + } + + clusterConfig := providers.BuildCapvClusterConfig(providers.ClusterConfig{ + Name: "${cluster_name}", + Organization: "${organization}", + }) + clusterValues, err := capv.GenerateClusterValues(clusterConfig) + + if err != nil { + return clusterBaseTemplates, err + } + + defaultAppsAppCr, err := generateDefaultAppsAppCrTemplate("default-apps-vsphere") + + if err != nil { + return clusterBaseTemplates, err + } + + defaultAppsValues, err := capv.GenerateDefaultAppsValues(capv.DefaultAppsConfig{ + ClusterName: "${cluster_name}", + Organization: "${organization}", + }) + + if err != nil { + return clusterBaseTemplates, err + } + + clusterBaseTemplates.ClusterAppCr = clusterAppCr + clusterBaseTemplates.ClusterValues = clusterValues + clusterBaseTemplates.DefaultAppsAppCr = defaultAppsAppCr + clusterBaseTemplates.DefaultAppsValues = defaultAppsValues + + return clusterBaseTemplates, nil +} + func generateCapZClusterBaseTemplates(structureConfig common.StructureConfig) (common.ClusterBaseTemplates, error) { clusterBaseTemplates := common.ClusterBaseTemplates{}