diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh index ab31acc794..e49843e55d 100755 --- a/parts/k8s/cloud-init/artifacts/cse_config.sh +++ b/parts/k8s/cloud-init/artifacts/cse_config.sh @@ -208,7 +208,9 @@ configureK8s() { "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", "maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT}, "providerKeyName": "k8s", - "providerKeyVersion": "" + "providerKeyVersion": "", + "enableMultipleStandardLoadBalancers": ${ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS}, + "tags": "${TAGS}" } EOF set -x diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 369974137e..a9d829731e 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -313,6 +313,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.CloudProviderDisableOutboundSNAT = vlabs.CloudProviderDisableOutboundSNAT api.KubeReservedCgroup = vlabs.KubeReservedCgroup api.MicrosoftAptRepositoryURL = vlabs.MicrosoftAptRepositoryURL + api.EnableMultipleStandardLoadBalancers = vlabs.EnableMultipleStandardLoadBalancers + api.Tags = vlabs.Tags convertComponentsToAPI(vlabs, api) convertAddonsToAPI(vlabs, api) convertKubeletConfigToAPI(vlabs, api) diff --git a/pkg/api/types.go b/pkg/api/types.go index a7307b6e90..44c955b1c4 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -441,91 +441,93 @@ const ( // KubernetesConfig contains the Kubernetes config structure, containing // Kubernetes specific configuration type KubernetesConfig struct { - KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` - KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` - MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` - ClusterSubnet string `json:"clusterSubnet,omitempty"` - NetworkPolicy string `json:"networkPolicy,omitempty"` - NetworkPlugin string `json:"networkPlugin,omitempty"` - NetworkMode string `json:"networkMode,omitempty"` - ContainerRuntime string `json:"containerRuntime,omitempty"` - MaxPods int `json:"maxPods,omitempty"` - DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` - DNSServiceIP string `json:"dnsServiceIP,omitempty"` - ServiceCIDR string `json:"serviceCidr,omitempty"` - UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` - UserAssignedID string `json:"userAssignedID,omitempty"` - UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. - CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` - CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` - CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` - CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` - CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated - MobyVersion string `json:"mobyVersion,omitempty"` - ContainerdVersion string `json:"containerdVersion,omitempty"` - LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` - CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager - UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` - CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` - WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` - WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` - WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` - UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` - EnableRbac *bool `json:"enableRbac,omitempty"` - EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` - EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` - PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` - GCHighThreshold int `json:"gchighthreshold,omitempty"` - GCLowThreshold int `json:"gclowthreshold,omitempty"` - EtcdVersion string `json:"etcdVersion,omitempty"` - EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` - EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` - EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` - EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` - EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` - EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` - Addons []KubernetesAddon `json:"addons,omitempty"` - Components []KubernetesComponent `json:"components,omitempty"` - KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` - ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig"` - ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` - CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` - APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` - SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated - KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` - CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` - CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` - CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` - CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` - CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` - CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` - CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` - CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` - CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` - CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` - CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` - NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` - NodeStatusUpdateFrequency string `json:"nodeStatusUpdateFrequency,omitempty"` - HardEvictionThreshold string `json:"hardEvictionThreshold,omitempty"` - CtrlMgrNodeMonitorGracePeriod string `json:"ctrlMgrNodeMonitorGracePeriod,omitempty"` - CtrlMgrPodEvictionTimeout string `json:"ctrlMgrPodEvictionTimeout,omitempty"` - CtrlMgrRouteReconciliationPeriod string `json:"ctrlMgrRouteReconciliationPeriod,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` - LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` - AzureCNIVersion string `json:"azureCNIVersion,omitempty"` - AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` - AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` - KeyVaultSku string `json:"keyVaultSku,omitempty"` - MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` - ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` - PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` - OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` - MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` + KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` + MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` + ClusterSubnet string `json:"clusterSubnet,omitempty"` + NetworkPolicy string `json:"networkPolicy,omitempty"` + NetworkPlugin string `json:"networkPlugin,omitempty"` + NetworkMode string `json:"networkMode,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` + MaxPods int `json:"maxPods,omitempty"` + DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` + ServiceCIDR string `json:"serviceCidr,omitempty"` + UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` + UserAssignedID string `json:"userAssignedID,omitempty"` + UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. + CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` + CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` + CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` + CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` + CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated + MobyVersion string `json:"mobyVersion,omitempty"` + ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` + CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` + WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` + UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` + EnableRbac *bool `json:"enableRbac,omitempty"` + EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` + EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` + PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` + GCHighThreshold int `json:"gchighthreshold,omitempty"` + GCLowThreshold int `json:"gclowthreshold,omitempty"` + EtcdVersion string `json:"etcdVersion,omitempty"` + EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` + EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` + EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` + EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` + Addons []KubernetesAddon `json:"addons,omitempty"` + Components []KubernetesComponent `json:"components,omitempty"` + KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` + ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig"` + ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` + SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` + PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` + CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` + CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` + CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` + CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` + CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` + CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` + CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` + NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` + NodeStatusUpdateFrequency string `json:"nodeStatusUpdateFrequency,omitempty"` + HardEvictionThreshold string `json:"hardEvictionThreshold,omitempty"` + CtrlMgrNodeMonitorGracePeriod string `json:"ctrlMgrNodeMonitorGracePeriod,omitempty"` + CtrlMgrPodEvictionTimeout string `json:"ctrlMgrPodEvictionTimeout,omitempty"` + CtrlMgrRouteReconciliationPeriod string `json:"ctrlMgrRouteReconciliationPeriod,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` + LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` + AzureCNIVersion string `json:"azureCNIVersion,omitempty"` + AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` + AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` + KeyVaultSku string `json:"keyVaultSku,omitempty"` + MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` + ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` + PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` + OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` + MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + EnableMultipleStandardLoadBalancers *bool `json:"enableMultipleStandardLoadBalancers,omitempty"` + Tags string `json:"tags,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest @@ -2348,62 +2350,64 @@ func (cs *ContainerService) GetProvisionScriptParametersCommon(input ProvisionSc cloudSpecConfig := cs.GetCloudSpecConfig() kubernetesConfig := cs.Properties.OrchestratorProfile.KubernetesConfig parameters := map[string]string{ - "ADMINUSER": cs.Properties.LinuxProfile.AdminUsername, - "ETCD_DOWNLOAD_URL": cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase, - "ETCD_VERSION": kubernetesConfig.EtcdVersion, - "CONTAINERD_VERSION": kubernetesConfig.ContainerdVersion, - "MOBY_VERSION": kubernetesConfig.MobyVersion, - "TENANT_ID": input.TenantID, - "KUBERNETES_VERSION": cs.Properties.GetKubernetesVersion(), - "HYPERKUBE_URL": cs.Properties.GetKubernetesHyperkubeSpec(), - "APISERVER_PUBLIC_KEY": input.APIServerCertificate, - "SUBSCRIPTION_ID": input.SubscriptionID, - "RESOURCE_GROUP": input.ResourceGroup, - "LOCATION": input.Location, - "VM_TYPE": cs.Properties.GetVMType(), - "SUBNET": cs.Properties.GetSubnetName(), - "NETWORK_SECURITY_GROUP": cs.Properties.GetNSGName(), - "VIRTUAL_NETWORK": cs.Properties.GetVirtualNetworkName(), - "VIRTUAL_NETWORK_RESOURCE_GROUP": cs.Properties.GetVNetResourceGroupName(), - "ROUTE_TABLE": cs.Properties.GetRouteTableName(), - "PRIMARY_AVAILABILITY_SET": cs.Properties.GetPrimaryAvailabilitySetName(), - "PRIMARY_SCALE_SET": cs.Properties.GetPrimaryScaleSetName(), - "SERVICE_PRINCIPAL_CLIENT_ID": input.ClientID, - "SERVICE_PRINCIPAL_CLIENT_SECRET": input.ClientSecret, - "KUBELET_PRIVATE_KEY": input.KubeletPrivateKey, - "NETWORK_PLUGIN": kubernetesConfig.NetworkPlugin, - "NETWORK_POLICY": kubernetesConfig.NetworkPolicy, - "VNET_CNI_PLUGINS_URL": kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig), - "CNI_PLUGINS_URL": cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL, - "CLOUDPROVIDER_BACKOFF": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderBackoff)), - "CLOUDPROVIDER_BACKOFF_MODE": kubernetesConfig.CloudProviderBackoffMode, - "CLOUDPROVIDER_BACKOFF_RETRIES": strconv.Itoa(kubernetesConfig.CloudProviderBackoffRetries), - "CLOUDPROVIDER_BACKOFF_EXPONENT": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64), - "CLOUDPROVIDER_BACKOFF_DURATION": strconv.Itoa(kubernetesConfig.CloudProviderBackoffDuration), - "CLOUDPROVIDER_BACKOFF_JITTER": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderRateLimit)), - "CLOUDPROVIDER_RATELIMIT_QPS": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT_QPS_WRITE": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPSWrite, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT_BUCKET": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucket), - "CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucketWrite), - "LOAD_BALANCER_DISABLE_OUTBOUND_SNAT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderDisableOutboundSNAT)), - "USE_MANAGED_IDENTITY_EXTENSION": strconv.FormatBool(to.Bool(kubernetesConfig.UseManagedIdentity)), - "USE_INSTANCE_METADATA": strconv.FormatBool(to.Bool(kubernetesConfig.UseInstanceMetadata)), - "LOAD_BALANCER_SKU": kubernetesConfig.LoadBalancerSku, - "EXCLUDE_MASTER_FROM_STANDARD_LB": strconv.FormatBool(to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)), - "MAXIMUM_LOADBALANCER_RULE_COUNT": strconv.Itoa(kubernetesConfig.MaximumLoadBalancerRuleCount), - "CONTAINER_RUNTIME": kubernetesConfig.ContainerRuntime, - "CONTAINERD_DOWNLOAD_URL_BASE": cloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, - "KMS_PROVIDER_VAULT_NAME": input.ClusterKeyVaultName, - "IPV6_DUALSTACK_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), - "IS_IPV6_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), - "AUTHENTICATION_METHOD": cs.Properties.GetCustomCloudAuthenticationMethod(), - "IDENTITY_SYSTEM": cs.Properties.GetCustomCloudIdentitySystem(), - "NETWORK_API_VERSION": APIVersionNetwork, - "NETWORK_MODE": kubernetesConfig.NetworkMode, - "KUBE_BINARY_URL": kubernetesConfig.CustomKubeBinaryURL, - "CUSTOM_HYPERKUBE_IMAGE": kubernetesConfig.CustomHyperkubeImage, - "MS_APT_REPO": kubernetesConfig.MicrosoftAptRepositoryURL, + "ADMINUSER": cs.Properties.LinuxProfile.AdminUsername, + "ETCD_DOWNLOAD_URL": cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase, + "ETCD_VERSION": kubernetesConfig.EtcdVersion, + "CONTAINERD_VERSION": kubernetesConfig.ContainerdVersion, + "MOBY_VERSION": kubernetesConfig.MobyVersion, + "TENANT_ID": input.TenantID, + "KUBERNETES_VERSION": cs.Properties.GetKubernetesVersion(), + "HYPERKUBE_URL": cs.Properties.GetKubernetesHyperkubeSpec(), + "APISERVER_PUBLIC_KEY": input.APIServerCertificate, + "SUBSCRIPTION_ID": input.SubscriptionID, + "RESOURCE_GROUP": input.ResourceGroup, + "LOCATION": input.Location, + "VM_TYPE": cs.Properties.GetVMType(), + "SUBNET": cs.Properties.GetSubnetName(), + "NETWORK_SECURITY_GROUP": cs.Properties.GetNSGName(), + "VIRTUAL_NETWORK": cs.Properties.GetVirtualNetworkName(), + "VIRTUAL_NETWORK_RESOURCE_GROUP": cs.Properties.GetVNetResourceGroupName(), + "ROUTE_TABLE": cs.Properties.GetRouteTableName(), + "PRIMARY_AVAILABILITY_SET": cs.Properties.GetPrimaryAvailabilitySetName(), + "PRIMARY_SCALE_SET": cs.Properties.GetPrimaryScaleSetName(), + "SERVICE_PRINCIPAL_CLIENT_ID": input.ClientID, + "SERVICE_PRINCIPAL_CLIENT_SECRET": input.ClientSecret, + "KUBELET_PRIVATE_KEY": input.KubeletPrivateKey, + "NETWORK_PLUGIN": kubernetesConfig.NetworkPlugin, + "NETWORK_POLICY": kubernetesConfig.NetworkPolicy, + "VNET_CNI_PLUGINS_URL": kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig), + "CNI_PLUGINS_URL": cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL, + "CLOUDPROVIDER_BACKOFF": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderBackoff)), + "CLOUDPROVIDER_BACKOFF_MODE": kubernetesConfig.CloudProviderBackoffMode, + "CLOUDPROVIDER_BACKOFF_RETRIES": strconv.Itoa(kubernetesConfig.CloudProviderBackoffRetries), + "CLOUDPROVIDER_BACKOFF_EXPONENT": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64), + "CLOUDPROVIDER_BACKOFF_DURATION": strconv.Itoa(kubernetesConfig.CloudProviderBackoffDuration), + "CLOUDPROVIDER_BACKOFF_JITTER": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderRateLimit)), + "CLOUDPROVIDER_RATELIMIT_QPS": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT_QPS_WRITE": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPSWrite, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT_BUCKET": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucket), + "CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucketWrite), + "LOAD_BALANCER_DISABLE_OUTBOUND_SNAT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderDisableOutboundSNAT)), + "USE_MANAGED_IDENTITY_EXTENSION": strconv.FormatBool(to.Bool(kubernetesConfig.UseManagedIdentity)), + "USE_INSTANCE_METADATA": strconv.FormatBool(to.Bool(kubernetesConfig.UseInstanceMetadata)), + "LOAD_BALANCER_SKU": kubernetesConfig.LoadBalancerSku, + "EXCLUDE_MASTER_FROM_STANDARD_LB": strconv.FormatBool(to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)), + "MAXIMUM_LOADBALANCER_RULE_COUNT": strconv.Itoa(kubernetesConfig.MaximumLoadBalancerRuleCount), + "CONTAINER_RUNTIME": kubernetesConfig.ContainerRuntime, + "CONTAINERD_DOWNLOAD_URL_BASE": cloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, + "KMS_PROVIDER_VAULT_NAME": input.ClusterKeyVaultName, + "IPV6_DUALSTACK_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), + "IS_IPV6_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), + "AUTHENTICATION_METHOD": cs.Properties.GetCustomCloudAuthenticationMethod(), + "IDENTITY_SYSTEM": cs.Properties.GetCustomCloudIdentitySystem(), + "NETWORK_API_VERSION": APIVersionNetwork, + "NETWORK_MODE": kubernetesConfig.NetworkMode, + "KUBE_BINARY_URL": kubernetesConfig.CustomKubeBinaryURL, + "CUSTOM_HYPERKUBE_IMAGE": kubernetesConfig.CustomHyperkubeImage, + "MS_APT_REPO": kubernetesConfig.MicrosoftAptRepositoryURL, + "TAGS": kubernetesConfig.Tags, + "ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS": strconv.FormatBool(to.Bool(kubernetesConfig.EnableMultipleStandardLoadBalancers)), } keys := make([]string, 0) diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 3eda2b6360..67faca01e8 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -323,85 +323,87 @@ const ( // KubernetesConfig contains the Kubernetes config structure, containing // Kubernetes specific configuration type KubernetesConfig struct { - KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` - KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` - MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` - ClusterSubnet string `json:"clusterSubnet,omitempty"` - DNSServiceIP string `json:"dnsServiceIP,omitempty"` - ServiceCidr string `json:"serviceCidr,omitempty"` - NetworkPolicy string `json:"networkPolicy,omitempty"` - NetworkPlugin string `json:"networkPlugin,omitempty"` - NetworkMode string `json:"networkMode,omitempty"` - ContainerRuntime string `json:"containerRuntime,omitempty"` - MaxPods int `json:"maxPods,omitempty"` - DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` - UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` - UserAssignedID string `json:"userAssignedID,omitempty"` - UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. - CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` - CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` - CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` - CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` - CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated - MobyVersion string `json:"mobyVersion,omitempty"` - ContainerdVersion string `json:"containerdVersion,omitempty"` - LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` - CustomCcmImage string `json:"customCcmImage,omitempty"` - UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` - CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` - WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` - WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` - WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` - UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` - EnableRbac *bool `json:"enableRbac,omitempty"` - EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` - EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` - PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` - GCHighThreshold int `json:"gchighthreshold,omitempty"` - GCLowThreshold int `json:"gclowthreshold,omitempty"` - EtcdVersion string `json:"etcdVersion,omitempty"` - EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` - EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` - EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` - EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` - EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` - EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` - Addons []KubernetesAddon `json:"addons,omitempty"` - Components []KubernetesComponent `json:"components,omitempty"` - ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"` - KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` - ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` - CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` - APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` - SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated - KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` - CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` - CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` - CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` - CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` - CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` - CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` - CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` - CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` - CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` - CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` - CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` - LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` - AzureCNIVersion string `json:"azureCNIVersion,omitempty"` - AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` - AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` - KeyVaultSku string `json:"keyVaultSku,omitempty"` - MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` - ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` - PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` - OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` - MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` + KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` + MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` + ClusterSubnet string `json:"clusterSubnet,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` + ServiceCidr string `json:"serviceCidr,omitempty"` + NetworkPolicy string `json:"networkPolicy,omitempty"` + NetworkPlugin string `json:"networkPlugin,omitempty"` + NetworkMode string `json:"networkMode,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` + MaxPods int `json:"maxPods,omitempty"` + DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` + UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` + UserAssignedID string `json:"userAssignedID,omitempty"` + UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. + CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` + CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` + CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` + CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` + CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated + MobyVersion string `json:"mobyVersion,omitempty"` + ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` + CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` + WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` + UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` + EnableRbac *bool `json:"enableRbac,omitempty"` + EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` + EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` + PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` + GCHighThreshold int `json:"gchighthreshold,omitempty"` + GCLowThreshold int `json:"gclowthreshold,omitempty"` + EtcdVersion string `json:"etcdVersion,omitempty"` + EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` + EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` + EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` + EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` + Addons []KubernetesAddon `json:"addons,omitempty"` + Components []KubernetesComponent `json:"components,omitempty"` + ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"` + KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` + ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` + SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` + PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` + CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` + CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` + CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` + CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` + CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` + CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` + CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` + LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` + AzureCNIVersion string `json:"azureCNIVersion,omitempty"` + AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` + AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` + KeyVaultSku string `json:"keyVaultSku,omitempty"` + MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` + ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` + PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` + OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` + MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + EnableMultipleStandardLoadBalancers *bool `json:"enableMultipleStandardLoadBalancers,omitempty"` + Tags string `json:"tags,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index eee192ed31..4305f79251 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1507,6 +1507,13 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack if e := k.validateKubernetesImageBaseType(); e != nil { return e } + + if to.Bool(k.EnableMultipleStandardLoadBalancers) && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") { + return errors.Errorf("OrchestratorProfile.KubernetesConfig.EnableMultipleStandardLoadBalancers is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion) + } + if k.Tags != "" && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") { + return errors.Errorf("OrchestratorProfile.KubernetesConfig.Tags is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion) + } return k.validateContainerRuntimeConfig() } diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index aec2552371..543c7624f3 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -706,6 +706,22 @@ func Test_KubernetesConfig_Validate(t *testing.T) { t.Error("should error when dual stack and single stack IPv6 enabled simultaneously") } } + + // Tests that apply to 1.20 and later releases + for _, k8sVersion := range common.GetVersionsLt(common.GetAllSupportedKubernetesVersions(false, false, false), "1.20.0", false, false) { + c := KubernetesConfig{ + EnableMultipleStandardLoadBalancers: to.BoolPtr(true), + } + if err := c.Validate(k8sVersion, false, false, false); err == nil { + t.Errorf("should error when enable multiple standard load balancer before v1.20.0") + } + c = KubernetesConfig{ + Tags: "a=b", + } + if err := c.Validate(k8sVersion, false, false, false); err == nil { + t.Errorf("should error when setting tags before v1.20.0") + } + } } func Test_Properties_ValidateCustomKubeComponent(t *testing.T) { diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index cfac91fc3d..48c74eec29 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -11796,7 +11796,9 @@ configureK8s() { "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", "maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT}, "providerKeyName": "k8s", - "providerKeyVersion": "" + "providerKeyVersion": "", + "enableMultipleStandardLoadBalancers": ${ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS}, + "tags": "${TAGS}" } EOF set -x