diff --git a/cloud-control-manager/cloud-driver/drivers/azure/connect/Azure_CloudConnection.go b/cloud-control-manager/cloud-driver/drivers/azure/connect/Azure_CloudConnection.go index 8ae229adf..3ae613764 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/connect/Azure_CloudConnection.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/connect/Azure_CloudConnection.go @@ -115,6 +115,7 @@ func (cloudConn *AzureCloudConnection) CreateVMHandler() (irs.VMHandler, error) Region: cloudConn.Region, Ctx: cloudConn.Ctx, Client: cloudConn.VMClient, + ScaleSetVMsClient: cloudConn.VirtualMachineScaleSetVMsClient, SubnetClient: cloudConn.SubnetClient, NicClient: cloudConn.VNicClient, PublicIPClient: cloudConn.PublicIPClient, diff --git a/cloud-control-manager/cloud-driver/drivers/azure/resources/ClusterHandler.go b/cloud-control-manager/cloud-driver/drivers/azure/resources/ClusterHandler.go index 14f2f0c33..0365d2f25 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/resources/ClusterHandler.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/resources/ClusterHandler.go @@ -959,8 +959,9 @@ func getNodeInfoStatus(agentPool armcontainerservice.AgentPool, virtualMachineSc return irs.NodeGroupInactive } -func getclusterDNSPrefix(clusterName string) string { - return fmt.Sprintf("%s-dns", clusterName) +func getclusterDNSPrefix() string { + prefix := "dns-" + return fmt.Sprintf("%s%d", prefix, time.Now().UnixNano()) } func getclusterNodeResourceGroup(clusterName string, resourceGroup string, regionString string) string { @@ -1135,7 +1136,7 @@ func createCluster(clusterReqInfo irs.ClusterInfo, ac *AzureClusterHandler) erro Properties: &armcontainerservice.ManagedClusterProperties{ KubernetesVersion: toStrPtr(clusterReqInfo.Version), EnableRBAC: toBoolPtr(true), - DNSPrefix: toStrPtr(getclusterDNSPrefix(clusterReqInfo.IId.NameId)), + DNSPrefix: toStrPtr(getclusterDNSPrefix()), NodeResourceGroup: toStrPtr(getclusterNodeResourceGroup(clusterReqInfo.IId.NameId, ac.Region.Region, ac.Region.Region)), AgentPoolProfiles: agentPoolProfiles, NetworkProfile: &networkProfile, @@ -1334,7 +1335,7 @@ func generatorClusterTags(sshKeyName string, clusterName string) (map[string]*st func getSSHKeyIIDByNodeGroups(NodeGroupInfos []irs.NodeGroupInfo) (irs.IID, error) { var key *irs.IID for _, nodeGroup := range NodeGroupInfos { - if nodeGroup.KeyPairIID.NameId != "" && nodeGroup.KeyPairIID.SystemId != "" { + if nodeGroup.KeyPairIID.NameId != "" || nodeGroup.KeyPairIID.SystemId != "" { key = &nodeGroup.KeyPairIID break } diff --git a/cloud-control-manager/cloud-driver/drivers/azure/resources/CommonAzureFunc.go b/cloud-control-manager/cloud-driver/drivers/azure/resources/CommonAzureFunc.go index 0350ee302..38557c619 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/resources/CommonAzureFunc.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/resources/CommonAzureFunc.go @@ -292,6 +292,19 @@ func GetDiskInfoType(diskType armcompute.DiskStorageAccountTypes) string { } } +func GetScaleSetVMDiskInfoType(diskType armcompute.StorageAccountTypes) string { + switch diskType { + case armcompute.StorageAccountTypesPremiumLRS: + return PremiumSSD + case armcompute.StorageAccountTypesStandardSSDLRS: + return StandardSSD + case armcompute.StorageAccountTypesStandardLRS: + return StandardHDD + default: + return string(diskType) + } +} + func overlapCheckCidr(cidr1 string, cidr2 string) (bool, error) { cidr1IP, cidr1IPnet, err := net.ParseCIDR(cidr1) if err != nil { diff --git a/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go b/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go index 8adfa2ef0..deffbef55 100644 --- a/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go +++ b/cloud-control-manager/cloud-driver/drivers/azure/resources/VMHandler.go @@ -47,6 +47,7 @@ type AzureVMHandler struct { Region idrv.RegionInfo Ctx context.Context Client *armcompute.VirtualMachinesClient + ScaleSetVMsClient *armcompute.VirtualMachineScaleSetVMsClient SubnetClient *armnetwork.SubnetsClient NicClient *armnetwork.InterfacesClient PublicIPClient *armnetwork.PublicIPAddressesClient @@ -876,7 +877,9 @@ func (vmHandler *AzureVMHandler) ListVM() ([]*irs.VMInfo, error) { return vmInfoList, nil } -func (vmHandler *AzureVMHandler) GetVM(vmIID irs.IID) (irs.VMInfo, error) { +func (vmHandler *AzureVMHandler) GetVM(vmIID irs.IID) (vmInfo irs.VMInfo, err error) { + vmInfo = irs.VMInfo{} + // log HisCall hiscallInfo := GetCallLogScheme(vmHandler.Region, call.VM, vmIID.NameId, "GetVM()") start := call.Start() @@ -885,21 +888,34 @@ func (vmHandler *AzureVMHandler) GetVM(vmIID irs.IID) (irs.VMInfo, error) { getErr := errors.New(fmt.Sprintf("Failed to Get VM. err = %s", err)) cblogger.Error(getErr.Error()) LoggingError(hiscallInfo, getErr) - return irs.VMInfo{}, getErr + return vmInfo, getErr } - vm, err := GetRawVM(convertedIID, vmHandler.Region.Region, vmHandler.Client, vmHandler.Ctx) - if err != nil { - getErr := errors.New(fmt.Sprintf("Failed to Get VM. err = %s", err)) - cblogger.Error(getErr.Error()) - LoggingError(hiscallInfo, getErr) - return irs.VMInfo{}, getErr + if strings.Contains(convertedIID.SystemId, "/virtualMachineScaleSets/") { + scaleSetVM, scaleSetVMResourceGroup, err := GetRawScaleSetVM(convertedIID.SystemId, vmHandler.CredentialInfo.SubscriptionId, vmHandler.ScaleSetVMsClient, vmHandler.Ctx) + if err != nil { + getErr := errors.New(fmt.Sprintf("Failed to Get VM. err = %s", err)) + cblogger.Error(getErr.Error()) + LoggingError(hiscallInfo, getErr) + return vmInfo, getErr + } + + vmInfo = vmHandler.mappingScaleSetServerInfo(scaleSetVM, scaleSetVMResourceGroup) + } else { + vm, err := GetRawVM(convertedIID, vmHandler.Region.Region, vmHandler.Client, vmHandler.Ctx) + if err != nil { + getErr := errors.New(fmt.Sprintf("Failed to Get VM. err = %s", err)) + cblogger.Error(getErr.Error()) + LoggingError(hiscallInfo, getErr) + return irs.VMInfo{}, getErr + } + + vmInfo = vmHandler.mappingServerInfo(vm) } // addAdministratorUser(convertedIID, vmHandler.Client, vmHandler.VirtualMachineRunCommandsClient, vmHandler.Ctx, vmHandler.Region) LoggingInfo(hiscallInfo, start) - vmInfo := vmHandler.mappingServerInfo(vm) return vmInfo, nil } @@ -1168,6 +1184,146 @@ func (vmHandler *AzureVMHandler) mappingServerInfo(server armcompute.VirtualMach return vmInfo } +func (vmHandler *AzureVMHandler) mappingScaleSetServerInfo(server armcompute.VirtualMachineScaleSetVM, resourceGroup string) irs.VMInfo { + // Get Default VM Info + vmInfo := irs.VMInfo{ + IId: irs.IID{ + NameId: *server.Name, + SystemId: *server.ID, + }, + Region: irs.RegionInfo{ + Region: *server.Location, + }, + VMSpecName: string(*server.Properties.HardwareProfile.VMSize), + RootDeviceName: "Not visible in Azure", + VMBlockDisk: "Not visible in Azure", + } + + // Set VM Zone + if server.Zones != nil && len(server.Zones) > 0 { + vmInfo.Region.Zone = *server.Zones[0] + } + + // Set VM Image Info + if reflect.ValueOf(server.Properties.StorageProfile.ImageReference.ID).IsNil() { + imageRef := server.Properties.StorageProfile.ImageReference + vmInfo.ImageIId.SystemId = *imageRef.Publisher + ":" + *imageRef.Offer + ":" + *imageRef.SKU + ":" + *imageRef.Version + vmInfo.ImageIId.NameId = *imageRef.Publisher + ":" + *imageRef.Offer + ":" + *imageRef.SKU + ":" + *imageRef.Version + //vmInfo.ImageIId.SystemId = vmInfo.ImageIId.NameId + } else { + vmInfo.ImageIId.SystemId = *server.Properties.StorageProfile.ImageReference.ID + vmInfo.ImageIId.NameId = *server.Properties.StorageProfile.ImageReference.ID + //vmInfo.ImageIId.SystemId = vmInfo.ImageIId.NameId + } + + // Get VNic ID + niList := server.Properties.NetworkProfile.NetworkInterfaces + var VNicId string + for _, ni := range niList { + if ni.ID != nil { + VNicId = *ni.ID + } + } + + // Get VNic + nicIdArr := strings.Split(VNicId, "/") + nicName := nicIdArr[len(nicIdArr)-1] + vmInfo.NetworkInterface = nicName + + for _, config := range server.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations { + if *config.Name == nicName { + // Get SecurityGroup + sgGroupIdArr := strings.Split(*config.Properties.NetworkSecurityGroup.ID, "/") + sgGroupName := sgGroupIdArr[len(sgGroupIdArr)-1] + vmInfo.SecurityGroupIIds = []irs.IID{ + { + NameId: sgGroupName, + SystemId: *config.Properties.NetworkSecurityGroup.ID, + }, + } + + for _, ip := range config.Properties.IPConfigurations { + if ip.Properties.Primary != nil && *ip.Properties.Primary { + // Get Subnet + subnetIdArr := strings.Split(*ip.Properties.Subnet.ID, "/") + subnetName := subnetIdArr[len(subnetIdArr)-1] + vmInfo.SubnetIID = irs.IID{NameId: subnetName, SystemId: *ip.Properties.Subnet.ID} + + // Get VPC + vpcIdArr := subnetIdArr[:len(subnetIdArr)-2] + vpcName := vpcIdArr[len(vpcIdArr)-1] + vmInfo.VpcIID = irs.IID{NameId: vpcName, SystemId: strings.Join(vpcIdArr, "/")} + } + } + } + } + + osType := getOSTypeByScaleSetVM(server) + if osType == irs.WINDOWS { + vmInfo.VMUserId = WindowBaseUser + } + if osType == irs.LINUX_UNIX { + vmInfo.VMUserId = CBVMUser + } + // Set GuestUser Id/Pwd + //if server.VirtualMachineProperties.OsProfile.AdminUsername != nil { + // vmInfo.VMUserId = *server.VirtualMachineProperties.OsProfile.AdminUsername + //} + if server.Properties.OSProfile.AdminPassword != nil { + vmInfo.VMUserPasswd = *server.Properties.OSProfile.AdminPassword + } + + // Set BootDisk + vmInfo.VMBootDisk = *server.Properties.StorageProfile.OSDisk.Name + vmInfo.RootDiskSize = strconv.Itoa(int(*server.Properties.StorageProfile.OSDisk.DiskSizeGB)) + vmInfo.RootDiskType = GetScaleSetVMDiskInfoType(*server.Properties.StorageProfile.OSDisk.ManagedDisk.StorageAccountType) + + // Get Keypair + keypairHandler := AzureKeyPairHandler{ + CredentialInfo: vmHandler.CredentialInfo, + Region: vmHandler.Region, + Ctx: vmHandler.Ctx, + Client: vmHandler.SshKeyClient, + } + keys, _ := keypairHandler.ListKey() + for _, key := range keys { + if server.Properties.OSProfile.LinuxConfiguration != nil { + for _, pubKey := range server.Properties.OSProfile.LinuxConfiguration.SSH.PublicKeys { + if key.PublicKey == *pubKey.KeyData { + vmInfo.KeyPairIId = key.IId + } + } + } + } + + if server.Properties.StorageProfile != nil && server.Properties.StorageProfile.DataDisks != nil && len(server.Properties.StorageProfile.DataDisks) > 0 { + dataDisks := server.Properties.StorageProfile.DataDisks + dataDiskIIDList := make([]irs.IID, len(dataDisks)) + for i, dataDisk := range dataDisks { + diskId := *dataDisk.ManagedDisk.ID + dataDiskIIDList[i] = irs.IID{ + NameId: GetResourceNameById(diskId), + SystemId: diskId, + } + } + vmInfo.DataDiskIIDs = dataDiskIIDList + } + osPlatform := getOSTypeByScaleSetVM(server) + vmInfo.Platform = osPlatform + if vmInfo.PublicIP != "" { + if osPlatform == irs.WINDOWS { + vmInfo.AccessPoint = fmt.Sprintf("%s:%s", vmInfo.PublicIP, "3389") + } else { + vmInfo.AccessPoint = fmt.Sprintf("%s:%s", vmInfo.PublicIP, "22") + } + } + if server.Tags != nil { + vmInfo.TagList = setTagList(server.Tags) + } + + return vmInfo +} + // VM 생성 시 Public IP 자동 생성 (nested flow 적용) func CreatePublicIP(vmHandler *AzureVMHandler, vmReqInfo irs.VMReqInfo) (irs.IID, error) { // PublicIP 이름 생성 @@ -1560,6 +1716,49 @@ func GetRawVM(vmIID irs.IID, resourceGroup string, client *armcompute.VirtualMac } } +func GetRawScaleSetVM(systemID string, subscriptionID string, client *armcompute.VirtualMachineScaleSetVMsClient, ctx context.Context) (vm armcompute.VirtualMachineScaleSetVM, resourceGroup string, err error) { + // systemID example + // /subscriptions/XXXXXXXX/resourceGroups/CB_XXXXXXXX/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodegroup0-XXXX-vmss/virtualMachines/0 + + vm = armcompute.VirtualMachineScaleSetVM{} + + var vmList []*armcompute.VirtualMachineScaleSetVM + + slist := strings.Split(systemID, "/") + if len(slist) < 3 { + return vm, resourceGroup, errors.New(fmt.Sprintf("Invalid systemID")) + } + vmScaleSetName := slist[len(slist)-3] + + cut := strings.ReplaceAll(systemID, "/subscriptions/"+subscriptionID+"/resourceGroups/", "") + slist = strings.Split(cut, "/") + if len(slist) < 1 { + return vm, resourceGroup, errors.New(fmt.Sprintf("Invalid systemID")) + } + resourceGroup = slist[0] + pager := client.NewListPager(resourceGroup, vmScaleSetName, nil) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return vm, resourceGroup, nil + } + + for _, vm := range page.Value { + vmList = append(vmList, vm) + } + } + + for _, v := range vmList { + if *v.ID == systemID { + vm = *v + return vm, resourceGroup, nil + } + } + notExistVpcErr := errors.New(fmt.Sprintf("The ScaleSetVM id %s not found", systemID)) + return vm, resourceGroup, notExistVpcErr +} + func generatePublicIPName(vmName string) string { rand.Seed(time.Now().UnixNano()) return fmt.Sprintf("%s-%s-PublicIP", vmName, strconv.FormatInt(rand.Int63n(100000), 10)) @@ -1656,11 +1855,18 @@ func resizeVMOsDisk(RootDiskSize string, vmReqIId irs.IID, resourceGroup string, func ConvertVMIID(vmIID irs.IID, credentialInfo idrv.CredentialInfo, regionInfo idrv.RegionInfo) (irs.IID, error) { if vmIID.NameId == "" && vmIID.SystemId == "" { - return vmIID, errors.New(fmt.Sprintf("nvalid IID")) + return vmIID, errors.New(fmt.Sprintf("Invalid IID")) } if vmIID.SystemId == "" { sysID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s", credentialInfo.SubscriptionId, regionInfo.Region, vmIID.NameId) return irs.IID{NameId: vmIID.NameId, SystemId: sysID}, nil + } else if strings.Contains(vmIID.SystemId, "/virtualMachineScaleSets/") { + slist := strings.Split(vmIID.SystemId, "/") + if len(slist) < 3 { + return vmIID, errors.New(fmt.Sprintf("Invalid IID")) + } + s := slist[len(slist)-3] + "_" + slist[len(slist)-1] + return irs.IID{NameId: s, SystemId: vmIID.SystemId}, nil } else { slist := strings.Split(vmIID.SystemId, "/") if len(slist) == 0 { @@ -1804,6 +2010,13 @@ func getOSTypeByVM(server armcompute.VirtualMachine) irs.Platform { return irs.WINDOWS } +func getOSTypeByScaleSetVM(server armcompute.VirtualMachineScaleSetVM) irs.Platform { + if server.Properties.OSProfile.LinuxConfiguration != nil { + return irs.LINUX_UNIX + } + return irs.WINDOWS +} + func getOSTypeByPublicImage(imageIID irs.IID) (irs.Platform, error) { if imageIID.NameId == "" && imageIID.SystemId == "" { return "", errors.New("failed get OSType By ImageIID err = empty ImageIID")