diff --git a/terraform/layer1-aws/.terraform.lock.hcl b/terraform/layer1-aws/.terraform.lock.hcl new file mode 100644 index 00000000..197d7fbe --- /dev/null +++ b/terraform/layer1-aws/.terraform.lock.hcl @@ -0,0 +1,105 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.1.0" + constraints = ">= 2.49.0, >= 3.0.0, >= 3.34.0, >= 3.35.0, >= 3.72.0, >= 4.0.0, >= 4.7.0, >= 4.35.0, >= 4.40.0, >= 4.47.0, 5.1.0" + hashes = [ + "h1:iDyYmwv8q94Dvr4DRG1KBxTWPZRFkRmKGa3cjCEsPZU=", + "zh:0c48f157b804c1f392adb5c14b81e756c652755e358096300ea8dd1283021129", + "zh:1a50495a6c0e5665e51df57dac6e781ec71439b11ebf05f971b6f3a3eb4eb7b2", + "zh:2959ff472c05e56d59e012118dd8d55022f005534c0ae961ce81136de9f66a4d", + "zh:2dfda9133581b99ed6e709e89a453fd2974ce88c703d3e073ec31bf99d7508ce", + "zh:2f3d92cc7a6624da42cee2202f8fb23e6d38f156ab7851884d637282cb0dc709", + "zh:3bc2a34d09cbaf439a1815846904f070c782cd8dfd60b5e0116827cda25f7549", + "zh:4ef43f1a247aa8de8690ac3bbc2b00ebaf6b2872fc8d0f5130e4a8130c874b87", + "zh:5477cb272dcaeb0030091bcf23a9f0f33b5410e44e317e9d3d49446f545dbaa4", + "zh:734c8fb4c0b79c82dd757566761dda5b91ee1ef9a2b848a748ade11e0e1cc69f", + "zh:80346c051b677f4f018da7fe06318b87c5bd0f1ec67ce78ab33baed3bb8b031a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a865b2f88dfee13df14116c5cf53d033d2c15855f4b59b9c65337309a928df2c", + "zh:c0345f266eedaece5612c1000722b302f895d1bc5af1d5a4265f0e7000ca48bb", + "zh:d59703c8e6a9d8b4fbd3b4583b945dfff9cb2844c762c0b3990e1cef18282279", + "zh:d8d04a6a6cd2dfcb23b57e551db7b15e647f6166310fb7d883d8ec67bdc9bdc8", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.3.3" + constraints = ">= 2.0.0" + hashes = [ + "h1:GmJ8PxLjjPr+lh02Bw3u7RYqA3UtpE2hQ1T43Vt7PTQ=", + "zh:0bd6ee14ca5cf0f0c83d3bb965346b1225ccd06a6247e80774aaaf54c729daa7", + "zh:3055ad0dcc98de1d4e45b72c5889ae91b62f4ae4e54dbc56c4821be0fdfbed91", + "zh:32764cfcff0d7379ca8b7dde376ac5551854d454c5881945f1952b785a312fa2", + "zh:55c2a4dc3ebdeaa1dec3a36db96dab253c7fa10b9fe1209862e1ee77a01e0aa1", + "zh:5c71f260ba5674d656d12f67cde3bb494498e6b6b6e66945ef85688f185dcf63", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9617280a853ec7caedb8beb7864e4b29faf9c850a453283980c28fccef2c493d", + "zh:ac8bda21950f8dddade3e9bc15f7bcfdee743738483be5724169943cafa611f5", + "zh:ba9ab567bbe63dee9197a763b3104ea9217ba27449ed54d3afa6657f412e3496", + "zh:effd1a7e34bae3879c02f03ed3afa979433a518e11de1f8afd35a8710231ac14", + "zh:f021538c86d0ac250d75e59efde6d869bbfff711eb744c8bddce79d2475bf46d", + "zh:f1e3984597948a2103391a26600e177b19f16a5a4c66acee27a4343fb141571f", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.19.0" + constraints = ">= 2.10.0, 2.19.0" + hashes = [ + "h1:ID/u9YOv00w+Z8iG+592oyuV7HcqRmPiZpEC9hnyTMY=", + "zh:028d346460de2d1d19b4c863dfc36be51c7bcd97d372b54a3a946bcb19f3f613", + "zh:391d0b38c455437d0a2ab1beb6ce6e1230aa4160bbae11c58b2810b258b44280", + "zh:40ea742f91b67f66e71d7091cfd40cc604528c4947651924bd6d8bd8d9793708", + "zh:48a99d341c8ba3cadaafa7cb99c0f11999f5e23f5cfb0f8469b4e352d9116e74", + "zh:4a5ade940eff267cbf7dcd52c1a7ac3999e7cc24996a409bd8b37bdb48a97f02", + "zh:5063742016a8249a4be057b9cc0ef24a684ec76d0ae5463d4b07e9b2d21e047e", + "zh:5d36b3a5662f840a6788f5e2a19d02139e87318feb3c5d82c7d076be1366fec4", + "zh:75edd9960cb30e54ef7de1b7df2761a274f17d4d41f54e72f86b43f41af3eb6d", + "zh:b85cadef3e6f25f1a10a617472bf5e8449decd61626733a1bc723de5edc08f64", + "zh:dc565b17b4ea6dde6bd1b92bc37e5e850fcbf9400540eec00ad3d9552a76ac2e", + "zh:deb665cc2123f2701aa3d653987b2ca35fb035a08a76a2382efb215c209f19a5", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.10.0" + constraints = ">= 0.9.0" + hashes = [ + "h1:NAl8eupFAZXCAbE5uiHZTz+Yqler55B3fMG+jNPrjjM=", + "zh:0ab31efe760cc86c9eef9e8eb070ae9e15c52c617243bbd9041632d44ea70781", + "zh:0ee4e906e28f23c598632eeac297ab098d6d6a90629d15516814ab90ad42aec8", + "zh:3bbb3e9da728b82428c6f18533b5b7c014e8ff1b8d9b2587107c966b985e5bcc", + "zh:6771c72db4e4486f2c2603c81dfddd9e28b6554d1ded2996b4cb37f887b467de", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:833c636d86c2c8f23296a7da5d492bdfd7260e22899fc8af8cc3937eb41a7391", + "zh:c545f1497ae0978ffc979645e594b57ff06c30b4144486f4f362d686366e2e42", + "zh:def83c6a85db611b8f1d996d32869f59397c23b8b78e39a978c8a2296b0588b2", + "zh:df9579b72cc8e5fac6efee20c7d0a8b72d3d859b50828b1c473d620ab939e2c7", + "zh:e281a8ecbb33c185e2d0976dc526c93b7359e3ffdc8130df7422863f4952c00e", + "zh:ecb1af3ae67ac7933b5630606672c94ec1f54b119bf77d3091f16d55ab634461", + "zh:f8109f13e07a741e1e8a52134f84583f97a819e33600be44623a21f6424d6593", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.5" + constraints = ">= 3.0.0" + hashes = [ + "h1:zeG5RmggBZW/8JWIVrdaeSJa0OG62uFX5HY1eE8SjzY=", + "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e", + "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32", + "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b", + "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a", + "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a", + "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e", + "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc", + "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9", + "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4", + "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8", + "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/layer1-aws/aws-eks.tf b/terraform/layer1-aws/aws-eks.tf index 440cac2d..b3e9e650 100644 --- a/terraform/layer1-aws/aws-eks.tf +++ b/terraform/layer1-aws/aws-eks.tf @@ -1,8 +1,4 @@ locals { - eks_worker_tags = { - "k8s.io/cluster-autoscaler/enabled" = "true" - "k8s.io/cluster-autoscaler/${local.name}" = "owned" - } eks_map_roles = [ { @@ -13,13 +9,14 @@ locals { ] } -data "aws_ami" "eks_default_bottlerocket" { +data "aws_ami" "eks_default_arm64" { most_recent = true owners = ["amazon"] filter { name = "name" - values = ["bottlerocket-aws-k8s-${var.eks_cluster_version}-x86_64-*"] + values = ["amazon-eks-arm64-node-${var.eks_cluster_version}-v*"] + } } @@ -65,7 +62,10 @@ module "eks" { cluster_endpoint_private_access = var.eks_cluster_endpoint_private_access cluster_endpoint_public_access_cidrs = var.eks_cluster_endpoint_only_pritunl ? ["${module.pritunl[0].pritunl_endpoint}/32"] : ["0.0.0.0/0"] + node_security_group_tags = { "karpenter.sh/discovery" = local.name } + self_managed_node_group_defaults = { + ami_id = data.aws_ami.eks_default_arm64.id block_device_mappings = { xvda = { device_name = "/dev/xvda" @@ -78,7 +78,7 @@ module "eks" { } } - iam_role_additional_policies = var.eks_workers_additional_policies + # iam_role_additional_policies = var.eks_workers_additional_policies metadata_options = { http_endpoint = "enabled" http_tokens = "required" @@ -88,81 +88,18 @@ module "eks" { iam_role_attach_cni_policy = false } self_managed_node_groups = { - spot = { - name = "${local.name}-spot" - iam_role_name = "${local.name}-spot" - desired_size = var.node_group_spot.desired_capacity - max_size = var.node_group_spot.max_capacity - min_size = var.node_group_spot.min_capacity - subnet_ids = module.vpc.private_subnets - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT --node-labels=nodegroup=spot'" - capacity_rebalance = var.node_group_spot.capacity_rebalance - use_mixed_instances_policy = var.node_group_spot.use_mixed_instances_policy - mixed_instances_policy = var.node_group_spot.mixed_instances_policy - - tags = local.eks_worker_tags - }, - ondemand = { - name = "${local.name}-ondemand" - iam_role_name = "${local.name}-ondemand" - desired_size = var.node_group_ondemand.desired_capacity - max_size = var.node_group_ondemand.max_capacity - min_size = var.node_group_ondemand.min_capacity - instance_type = var.node_group_ondemand.instance_type - subnet_ids = module.vpc.private_subnets - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=ON_DEMAND --node-labels=nodegroup=ondemand'" - capacity_rebalance = var.node_group_ondemand.capacity_rebalance - use_mixed_instances_policy = var.node_group_ondemand.use_mixed_instances_policy - mixed_instances_policy = var.node_group_ondemand.mixed_instances_policy - - tags = local.eks_worker_tags - }, - ci = { - name = "${local.name}-ci" - iam_role_name = "${local.name}-ci" - desired_size = var.node_group_ci.desired_capacity - max_size = var.node_group_ci.max_capacity - min_size = var.node_group_ci.min_capacity - subnet_ids = module.vpc.private_subnets - - bootstrap_extra_args = "--kubelet-extra-args '--node-labels=eks.amazonaws.com/capacityType=SPOT --node-labels=nodegroup=ci --register-with-taints=nodegroup=ci:NoSchedule'" - capacity_rebalance = var.node_group_ci.capacity_rebalance - use_mixed_instances_policy = var.node_group_ci.use_mixed_instances_policy - mixed_instances_policy = var.node_group_ci.mixed_instances_policy - - tags = merge(local.eks_worker_tags, { "k8s.io/cluster-autoscaler/node-template/label/nodegroup" = "ci" }) - }, - bottlerocket = { - name = "${local.name}-bottlerocket" - iam_role_name = "${local.name}-bottlerocket" - desired_size = var.node_group_br.desired_capacity - max_size = var.node_group_br.max_capacity - min_size = var.node_group_br.min_capacity + default = { + name = "${local.name}-default" + iam_role_name = "${local.name}-default" + desired_size = var.node_group_default.desired_capacity + max_size = var.node_group_default.max_capacity + min_size = var.node_group_default.min_capacity subnet_ids = module.vpc.private_subnets - platform = "bottlerocket" - ami_id = data.aws_ami.eks_default_bottlerocket.id - bootstrap_extra_args = <<-EOT - [settings.host-containers.admin] - enabled = false - - [settings.host-containers.control] - enabled = true - - [settings.kubernetes.node-labels] - "eks.amazonaws.com/capacityType" = "SPOT" - "nodegroup" = "bottlerocket" - - [settings.kubernetes.node-taints] - "nodegroup" = "bottlerocket:NoSchedule" - EOT - capacity_rebalance = var.node_group_br.capacity_rebalance - use_mixed_instances_policy = var.node_group_br.use_mixed_instances_policy - mixed_instances_policy = var.node_group_br.mixed_instances_policy - - tags = merge(local.eks_worker_tags, { "k8s.io/cluster-autoscaler/node-template/label/nodegroup" = "bottlerocket" }) + bootstrap_extra_args = "--kubelet-extra-args '--node-labels=nodegroup=default --register-with-taints=CriticalAddonsOnly=true:NoSchedule'" + capacity_rebalance = var.node_group_default.capacity_rebalance + use_mixed_instances_policy = var.node_group_default.use_mixed_instances_policy + mixed_instances_policy = var.node_group_default.mixed_instances_policy } } fargate_profiles = { diff --git a/terraform/layer1-aws/demo.tfvars.example b/terraform/layer1-aws/demo.tfvars.example index ff881fb3..65b67fc3 100644 --- a/terraform/layer1-aws/demo.tfvars.example +++ b/terraform/layer1-aws/demo.tfvars.example @@ -19,14 +19,3 @@ single_nat_gateway = true # EKS ########## eks_cluster_encryption_config_enable = true - -node_group_ondemand = { - instance_types = ["m5a.medium"] - capacity_type = "ON_DEMAND" - max_capacity = 5 - min_capacity = 1 - desired_capacity = 1 - force_update_version = false -} - -eks_write_kubeconfig = false diff --git a/terraform/layer1-aws/outputs.tf b/terraform/layer1-aws/outputs.tf index 0b29f5b4..10a1ebc6 100644 --- a/terraform/layer1-aws/outputs.tf +++ b/terraform/layer1-aws/outputs.tf @@ -108,3 +108,15 @@ output "ssl_certificate_arn" { description = "ARN of SSL certificate" value = local.ssl_certificate_arn } + +output "node_group_default_iam_instance_profile_id" { + value = module.eks.self_managed_node_groups["default"].iam_instance_profile_id +} + +output "node_group_default_iam_role_arn" { + value = module.eks.self_managed_node_groups["default"].iam_role_arn +} + +output "node_group_default_iam_role_name" { + value = module.eks.self_managed_node_groups["default"].iam_role_name +} diff --git a/terraform/layer1-aws/variables.tf b/terraform/layer1-aws/variables.tf index 3d98e246..2b32f86f 100644 --- a/terraform/layer1-aws/variables.tf +++ b/terraform/layer1-aws/variables.tf @@ -22,7 +22,7 @@ variable "aws_account_password_policy" { } variable "is_this_payment_account" { - default = true + default = false description = "Set it to false if a target account isn't a payer account. This variable is used to apply a configuration for cost allocation tags" } @@ -117,15 +117,15 @@ variable "eks_cluster_version" { description = "Version of the EKS K8S cluster" } -variable "eks_workers_additional_policies" { - type = map(string) - default = { - additional = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" - } - description = "Additional IAM policy attached to EKS worker nodes" -} +# variable "eks_workers_additional_policies" { +# type = map(string) +# default = { +# additional = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +# } +# description = "Additional IAM policy attached to EKS worker nodes" +# } -variable "node_group_spot" { +variable "node_group_default" { type = object({ instance_type = string max_capacity = number @@ -137,10 +137,10 @@ variable "node_group_spot" { }) default = { - instance_type = "t3.medium" # will be overridden - max_capacity = 5 - min_capacity = 0 - desired_capacity = 1 + instance_type = "t4g.medium" # will be overridden + max_capacity = 3 + min_capacity = 2 + desired_capacity = 2 capacity_rebalance = true use_mixed_instances_policy = true mixed_instances_policy = { @@ -150,101 +150,12 @@ variable "node_group_spot" { } override = [ - { instance_type = "t3.medium" }, - { instance_type = "t3a.medium" } - ] - } - } - description = "Spot node group configuration" -} - -variable "node_group_ci" { - type = object({ - instance_type = string - max_capacity = number - min_capacity = number - desired_capacity = number - capacity_rebalance = bool - use_mixed_instances_policy = bool - mixed_instances_policy = any - }) - - default = { - instance_type = "t3.medium" # will be overridden - max_capacity = 5 - min_capacity = 0 - desired_capacity = 0 - capacity_rebalance = false - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 0 - } - - override = [ - { instance_type = "t3.medium" }, - { instance_type = "t3a.medium" } + { instance_type = "t4g.small" }, + { instance_type = "t4g.medium" } ] } } - description = "CI node group configuration" -} - -variable "node_group_ondemand" { - type = object({ - instance_type = string - max_capacity = number - min_capacity = number - desired_capacity = number - capacity_rebalance = bool - use_mixed_instances_policy = bool - mixed_instances_policy = any - }) - - default = { - instance_type = "t3a.medium" - max_capacity = 5 - min_capacity = 1 - desired_capacity = 1 - capacity_rebalance = false - use_mixed_instances_policy = false - mixed_instances_policy = null - } - description = "Default ondemand node group configuration" -} - -variable "node_group_br" { - type = object({ - instance_type = string - max_capacity = number - min_capacity = number - desired_capacity = number - capacity_rebalance = bool - use_mixed_instances_policy = bool - mixed_instances_policy = any - }) - - default = { - instance_type = "t3.medium" # will be overridden - max_capacity = 5 - min_capacity = 0 - desired_capacity = 0 - capacity_rebalance = true - use_mixed_instances_policy = true - mixed_instances_policy = { - instances_distribution = { - on_demand_base_capacity = 0 - on_demand_percentage_above_base_capacity = 0 - } - - override = [ - { instance_type = "t3.medium" }, - { instance_type = "t3a.medium" } - ] - } - } - description = "Bottlerocket node group configuration" + description = "Default node group configuration" } variable "eks_map_roles" { @@ -258,11 +169,6 @@ variable "eks_map_roles" { default = [] } -variable "eks_write_kubeconfig" { - default = false - description = "Flag for eks module to write kubeconfig" -} - variable "eks_cluster_enabled_log_types" { type = list(string) default = ["audit"] diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf index 89396f82..ae21396d 100644 --- a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf +++ b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf @@ -19,16 +19,6 @@ serviceAccount: create: true annotations: "eks.amazonaws.com/role-arn": ${local.aws_load_balancer_controller.enabled ? module.aws_iam_aws_loadbalancer_controller[0].role_arn : ""} - -affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/capacityType - operator: In - values: - - ON_DEMAND VALUES } diff --git a/terraform/layer2-k8s/eks-aws-node-termination-handler.tf b/terraform/layer2-k8s/eks-aws-node-termination-handler.tf deleted file mode 100644 index d7f62101..00000000 --- a/terraform/layer2-k8s/eks-aws-node-termination-handler.tf +++ /dev/null @@ -1,93 +0,0 @@ -locals { - aws_node_termination_handler = { - name = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].id - enabled = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].enabled - chart = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].chart - repository = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].repository - chart_version = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].chart_version - namespace = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].namespace - } - aws_node_termination_handler_values = <