From 76da0139ba52f3b4b8563c8170d2fdbfa63d6306 Mon Sep 17 00:00:00 2001
From: Gordon Byers <gordonbyers@gmail.com>
Date: Thu, 26 Oct 2023 11:58:58 +0100
Subject: [PATCH] Spot Instances for user pools (#611)

* natgw not preview

* spot bicep

* spot

* userpool only

* maxsurge

---------

Co-authored-by: Gordon Byers <gordon.byers@microsoft.com>
---
 bicep/aksagentpool.bicep            | 57 +++++++++++++++++------------
 bicep/main.bicep                    |  4 ++
 helper/src/components/clusterTab.js |  1 +
 helper/src/components/deployTab.js  |  1 +
 helper/src/config.json              |  3 +-
 5 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/bicep/aksagentpool.bicep b/bicep/aksagentpool.bicep
index af865c8ed..d31a2b420 100644
--- a/bicep/aksagentpool.bicep
+++ b/bicep/aksagentpool.bicep
@@ -46,11 +46,20 @@ param osSKU string
 @description('Assign a public IP per node')
 param enableNodePublicIP bool = false
 
+@description('If the node pool should use VM spot instances')
+param spotInstance bool = false
+
 @description('Apply a default sku taint to Windows node pools')
 param autoTaintWindows bool = false
 
 var taints = autoTaintWindows ? union(nodeTaints, ['sku=Windows:NoSchedule']) : nodeTaints
 
+var spotProperties = {
+  scaleSetPriority: 'Spot'
+  scaleSetEvictionPolicy: 'Delete'
+  spotMaxPrice: -1
+}
+
 resource aks 'Microsoft.ContainerService/managedClusters@2023-08-02-preview' existing = {
   name: AksName
 }
@@ -58,27 +67,29 @@ resource aks 'Microsoft.ContainerService/managedClusters@2023-08-02-preview' exi
 resource userNodepool 'Microsoft.ContainerService/managedClusters/agentPools@2023-08-02-preview' = {
   parent: aks
   name: PoolName
-  properties: {
-    mode: 'User'
-    vmSize: agentVMSize
-    count: agentCount
-    minCount: autoScale ? agentCount : null
-    maxCount: autoScale ? agentCountMax : null
-    enableAutoScaling: autoScale
-    availabilityZones: !empty(availabilityZones) ? availabilityZones : null
-    osDiskType: osDiskType
-    osSKU: osSKU
-    osDiskSizeGB: osDiskSizeGB
-    osType: osType
-    maxPods: maxPods
-    type: 'VirtualMachineScaleSets'
-    vnetSubnetID: !empty(subnetId) ? subnetId : null
-    podSubnetID: !empty(podSubnetID) ? podSubnetID : null
-    upgradeSettings: {
-      maxSurge: '33%'
-    }
-    nodeTaints: taints
-    nodeLabels: nodeLabels
-    enableNodePublicIP: enableNodePublicIP
-  }
+  properties: union({
+      mode: 'User'
+      vmSize: agentVMSize
+      count: agentCount
+      minCount: autoScale ? agentCount : null
+      maxCount: autoScale ? agentCountMax : null
+      enableAutoScaling: autoScale
+      availabilityZones: !empty(availabilityZones) ? availabilityZones : null
+      osDiskType: osDiskType
+      osSKU: osSKU
+      osDiskSizeGB: osDiskSizeGB
+      osType: osType
+      maxPods: maxPods
+      type: 'VirtualMachineScaleSets'
+      vnetSubnetID: !empty(subnetId) ? subnetId : null
+      podSubnetID: !empty(podSubnetID) ? podSubnetID : null
+      upgradeSettings: spotInstance ? {} : {
+        maxSurge:  '33%' //Spot pools can't set max surge
+      }
+      nodeTaints: taints
+      nodeLabels: nodeLabels
+      enableNodePublicIP: enableNodePublicIP
+    },
+    spotInstance ? spotProperties : {}
+  )
 }
diff --git a/bicep/main.bicep b/bicep/main.bicep
index f72043022..e0f4d1eef 100644
--- a/bicep/main.bicep
+++ b/bicep/main.bicep
@@ -921,6 +921,9 @@ var autoScale = agentCountMax > agentCount
 @description('Name for user node pool')
 param nodePoolName string = 'npuser01'
 
+@description('Config the user node pool as a spot instance')
+param nodePoolSpot bool = false
+
 @description('Allocate pod ips dynamically')
 param cniDynamicIpAllocation bool = false
 
@@ -1396,6 +1399,7 @@ module userNodePool '../bicep/aksagentpool.bicep' = if (!JustUseSystemPool){
     enableNodePublicIP: enableNodePublicIP
     osDiskSizeGB: osDiskSizeGB
     availabilityZones: availabilityZones
+    spotInstance: nodePoolSpot
   }
 }
 
diff --git a/helper/src/components/clusterTab.js b/helper/src/components/clusterTab.js
index 3f59ca1cd..4abbc04ea 100644
--- a/helper/src/components/clusterTab.js
+++ b/helper/src/components/clusterTab.js
@@ -223,6 +223,7 @@ export default function ({ defaults, tabValues, updateFn, featureFlag, invalidAr
                                 <MessageBar messageBarType={MessageBarType.error}>{getError(invalidArray, 'osDiskType')}</MessageBar>
                             }
                             <TextField label="VM SKU" onChange={(ev, val) => updateFn('vmSize', val)} required errorMessage={getError(invalidArray, 'vmSize')} value={cluster.vmSize} />
+                            <Checkbox checked={cluster.nodePoolSpot} onChange={(ev, val) => updateFn("nodePoolSpot", val)} disabled={cluster.SystemPoolType=='none'} onRenderLabel={() => <Text styles={{ root: { color: 'gray' } }}>Spot Instance</Text>} />
                             <ChoiceGroup
                                 onChange={(ev, { key }) => updateFn("osDiskType", key)}
                                 selectedKey={cluster.osDiskType}
diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js
index 0b514540e..d08a2654c 100644
--- a/helper/src/components/deployTab.js
+++ b/helper/src/components/deployTab.js
@@ -40,6 +40,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
     ...(cluster.autoscale && { agentCountMax: cluster.maxCount }),
     ...(cluster.osType !== defaults.cluster.osType && { osType: cluster.osType}),
     ...(cluster.osSKU !== defaults.cluster.osSKU && { osSKU: cluster.osSKU}),
+    ...(cluster.SystemPoolType !== 'none' && cluster.nodePoolSpot !== defaults.cluster.nodePoolSpot && { nodePoolSpot: cluster.nodePoolSpot}),
     ...(cluster.osDiskType === "Managed" && { osDiskType: cluster.osDiskType, ...(cluster.osDiskSizeGB > 0 && { osDiskSizeGB: cluster.osDiskSizeGB }) }),
     ...(net.vnet_opt === "custom" && {
          custom_vnet: true,
diff --git a/helper/src/config.json b/helper/src/config.json
index 3c92aceda..55672feb1 100644
--- a/helper/src/config.json
+++ b/helper/src/config.json
@@ -59,7 +59,8 @@
             "enableAzureRBAC": true,
             "aadgroupids": "",
             "availabilityZones": "no",
-            "DefenderForContainers" : false
+            "DefenderForContainers" : false,
+            "nodePoolSpot": false
         },
         "addons": {
             "logDataCap": 0,