From 321908f694ccade2fc85935c808fd58e73798d24 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Wed, 29 Nov 2023 12:04:00 -0600 Subject: [PATCH 01/15] needs to be null to allow using the subnet default --- modules/nodepool/variables.tf | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nodepool/variables.tf b/modules/nodepool/variables.tf index 7610e1b..b666207 100644 --- a/modules/nodepool/variables.tf +++ b/modules/nodepool/variables.tf @@ -98,7 +98,7 @@ variable "spot" { } variable "associate_public_ip_address" { - default = false + default = null type = bool } diff --git a/variables.tf b/variables.tf index 644b5d7..b6dd398 100644 --- a/variables.tf +++ b/variables.tf @@ -193,7 +193,7 @@ variable "wait_for_capacity_timeout" { } variable "associate_public_ip_address" { - default = false + default = null type = bool } From f688373543e9ecd16edf26b932b1710ac0dbc1a1 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Wed, 29 Nov 2023 12:04:55 -0600 Subject: [PATCH 02/15] use some terraform functions available instead of winging it and remove the invalid old AMIs --- examples/cloud-enabled/main.tf | 89 +++++++++------------------------- 1 file changed, 22 insertions(+), 67 deletions(-) diff --git a/examples/cloud-enabled/main.tf b/examples/cloud-enabled/main.tf index aa50f74..8ac4bce 100644 --- a/examples/cloud-enabled/main.tf +++ b/examples/cloud-enabled/main.tf @@ -1,10 +1,17 @@ provider "aws" { region = local.aws_region + default_tags { + tags = local.tags + } } locals { cluster_name = "cloud-enabled" aws_region = "us-gov-west-1" + cidr = "10.88.0.0/16" + ssh_allowed_cidrs = [ + "0.0.0.0/0" + ] tags = { "terraform" = "true", @@ -12,21 +19,6 @@ locals { } } -data "aws_ami" "rhel7" { - most_recent = true - owners = ["219670896067"] # owner is specific to aws gov cloud - - filter { - name = "name" - values = ["RHEL-7*"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } -} - data "aws_ami" "rhel8" { most_recent = true owners = ["219670896067"] # owner is specific to aws gov cloud @@ -42,43 +34,13 @@ data "aws_ami" "rhel8" { } } -data "aws_ami" "centos7" { - most_recent = true - owners = ["345084742485"] # owner is specific to aws gov cloud - - filter { - name = "name" - values = ["CentOS Linux 7 x86_64 HVM EBS*"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } -} - -data "aws_ami" "centos8" { - most_recent = true - owners = ["345084742485"] # owner is specific to aws gov cloud - - filter { - name = "name" - values = ["CentOS Linux 8 x86_64 HVM EBS*"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } -} - # Key Pair resource "tls_private_key" "ssh" { algorithm = "RSA" rsa_bits = 4096 } -resource "local_file" "ssh_pem" { +resource "local_sensitive_file" "ssh_pem" { filename = "${local.cluster_name}.pem" content = tls_private_key.ssh.private_key_pem file_permission = "0600" @@ -91,11 +53,11 @@ module "vpc" { source = "terraform-aws-modules/vpc/aws" name = "rke2-${local.cluster_name}" - cidr = "10.88.0.0/16" + cidr = local.cidr azs = ["${local.aws_region}a", "${local.aws_region}b", "${local.aws_region}c"] - public_subnets = ["10.88.1.0/24", "10.88.2.0/24", "10.88.3.0/24"] - private_subnets = ["10.88.101.0/24", "10.88.102.0/24", "10.88.103.0/24"] + public_subnets = [cidrsubnet(local.cidr, 8, 1), cidrsubnet(local.cidr, 8, 2), cidrsubnet(local.cidr, 8, 3)] + private_subnets = [cidrsubnet(local.cidr, 8, 101), cidrsubnet(local.cidr, 8, 102), cidrsubnet(local.cidr, 8, 103)] enable_nat_gateway = true single_nat_gateway = true @@ -103,6 +65,11 @@ module "vpc" { enable_dns_hostnames = true enable_dns_support = true + # Note that EC2s launched into a public subnet that do not have a public IP + # address will not be able to access the Internet because public subnets + # do not get NAT gateways + map_public_ip_on_launch = true + # Add in required tags for proper AWS CCM integration public_subnet_tags = merge({ "kubernetes.io/cluster/${module.rke2.cluster_name}" = "shared" @@ -131,20 +98,14 @@ module "rke2" { ami = data.aws_ami.rhel8.image_id # Note: Multi OS is primarily for example purposes ssh_authorized_keys = [tls_private_key.ssh.public_key_openssh] - instance_type = "t3a.medium" + instance_type = "t3.medium" controlplane_internal = false # Note this defaults to best practice of true, but is explicitly set to public for demo purposes servers = 1 # Enable AWS Cloud Controller Manager enable_ccm = true - rke2_config = <<-EOT -node-label: - - "name=server" - - "os=rhel8" -EOT - - tags = local.tags + rke2_config = yamlencode({ "node-label" : ["name=server", "os=rhel8"] }) } # @@ -157,25 +118,19 @@ module "agents" { vpc_id = module.vpc.vpc_id subnets = module.vpc.public_subnets # Note: Public subnets used for demo purposes, this is not recommended in production - ami = data.aws_ami.rhel8.image_id # Note: Multi OS is primarily for example purposes + ami = data.aws_ami.rhel8.image_id ssh_authorized_keys = [tls_private_key.ssh.public_key_openssh] spot = true asg = { min : 1, max : 10, desired : 2 } - instance_type = "t3a.large" + instance_type = "t3.large" # Enable AWS Cloud Controller Manager and Cluster Autoscaler enable_ccm = true enable_autoscaler = true - rke2_config = <<-EOT -node-label: - - "name=generic" - - "os=rhel8" -EOT + rke2_config = yamlencode({ "node-label" : ["name=generic", "os=rhel8"] }) cluster_data = module.rke2.cluster_data - - tags = local.tags } # For demonstration only, lock down ssh access in production @@ -185,7 +140,7 @@ resource "aws_security_group_rule" "quickstart_ssh" { protocol = "tcp" security_group_id = module.rke2.cluster_data.cluster_sg type = "ingress" - cidr_blocks = ["0.0.0.0/0"] + cidr_blocks = local.ssh_allowed_cidrs } # Generic outputs as examples From e066c4604ca189a9fc4cffa15ebb1898bdee0886 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 5 Dec 2023 09:28:57 -0600 Subject: [PATCH 03/15] use stable channel instead of pinning version by default because it's getting quite old and don't want to periodically update it --- data.tf | 11 +++++++++++ modules/agent-nodepool/main.tf | 8 +++++--- modules/agent-nodepool/variables.tf | 10 ++++++++-- modules/common/download.sh | 9 +++++++-- variables.tf | 10 ++++++++-- 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/data.tf b/data.tf index da374b9..b677565 100644 --- a/data.tf +++ b/data.tf @@ -26,11 +26,13 @@ data "cloudinit_config" "this" { extra_cloud_config_config = var.extra_cloud_config_config }) } + part { filename = "00_pre.sh" content_type = "text/x-shellscript" content = module.init.pre_templated } + dynamic "part" { for_each = var.download ? [1] : [] content { @@ -38,6 +40,7 @@ data "cloudinit_config" "this" { content_type = "text/x-shellscript" content = templatefile("${path.module}/modules/common/download.sh", { # Must not use `version` here since that is reserved + rke2_channel = var.rke2_channel rke2_version = var.rke2_version type = "server" rke2_install_script_url = var.rke2_install_script_url @@ -52,6 +55,7 @@ data "cloudinit_config" "this" { content_type = "text/x-shellscript" content = module.init.rke2_templated } + part { filename = "99_post.sh" content_type = "text/x-shellscript" @@ -143,3 +147,10 @@ data "aws_iam_policy_document" "aws_ccm" { ] } } + +# Need to add getter/setter for statestore if provided with role +data "aws_iam_role" "provided" { + count = var.iam_instance_profile == "" ? 0 : 1 + + name = var.iam_instance_profile +} diff --git a/modules/agent-nodepool/main.tf b/modules/agent-nodepool/main.tf index 814f81a..e5d8f8e 100644 --- a/modules/agent-nodepool/main.tf +++ b/modules/agent-nodepool/main.tf @@ -84,6 +84,7 @@ data "cloudinit_config" "init" { extra_cloud_config_config = var.extra_cloud_config_config }) } + part { filename = "00_pre.sh" content_type = "text/x-shellscript" @@ -97,9 +98,9 @@ data "cloudinit_config" "init" { content_type = "text/x-shellscript" content = templatefile("${path.module}/../common/download.sh", { # Must not use `version` here since that is reserved - rke2_version = var.rke2_version - type = "agent" - + rke2_channel = var.rke2_channel + rke2_version = var.rke2_version + type = "agent" rke2_install_script_url = var.rke2_install_script_url awscli_url = var.awscli_url unzip_rpm_url = var.unzip_rpm_url @@ -113,6 +114,7 @@ data "cloudinit_config" "init" { content_type = "text/x-shellscript" content = module.init.rke2_templated } + part { filename = "99_post.sh" content_type = "text/x-shellscript" diff --git a/modules/agent-nodepool/variables.tf b/modules/agent-nodepool/variables.tf index c3bd8c9..b259c4a 100644 --- a/modules/agent-nodepool/variables.tf +++ b/modules/agent-nodepool/variables.tf @@ -135,10 +135,16 @@ variable "cluster_data" { }) } +variable "rke2_channel" { + description = "Channel to use for RKE2 agent nodepool" + type = string + default = null +} + variable "rke2_version" { - description = "Version to use for RKE2 server nodepool" + description = "Version to use for RKE2 agent nodepool" type = string - default = "v1.19.7+rke2r1" + default = null } variable "rke2_config" { diff --git a/modules/common/download.sh b/modules/common/download.sh index 1d551f0..d4dc962 100644 --- a/modules/common/download.sh +++ b/modules/common/download.sh @@ -2,7 +2,12 @@ set -e export INSTALL_RKE2_TYPE="${type}" +%{ if rke2_channel != null ~} +export INSTALL_RKE2_CHANNEL="${rke2_channel}" +%{ endif ~} +%{ if rke2_version != null ~} export INSTALL_RKE2_VERSION="${rke2_version}" +%{ endif ~} if [ "$${DEBUG}" == 1 ]; then set -x @@ -66,7 +71,7 @@ do_download() { case $ID in centos | rocky | rhel) info "Installing RKE2 for EL-based distro" - + install_unzip_el install_awscli @@ -99,7 +104,7 @@ do_download() { hostnamectl set-hostname $HOSTNAME INSTALL_RKE2_METHOD='tar' INSTALL_RKE2_TYPE="${type}" ./install.sh - + install_awscli ;; amzn) diff --git a/variables.tf b/variables.tf index b6dd398..a72152e 100644 --- a/variables.tf +++ b/variables.tf @@ -144,10 +144,16 @@ variable "metadata_options" { # # RKE2 Variables # +variable "rke2_channel" { + description = "Channel to use for RKE2 server nodepool" + type = string + default = null +} + variable "rke2_version" { - description = "Version to use for RKE2 server nodes" + description = "Version to use for RKE2 server nodepool" type = string - default = "v1.19.7+rke2r1" + default = null } variable "rke2_config" { From 689328dc6112fb7ce83530e3513ab71ea9d3c5da Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 5 Dec 2023 09:31:15 -0600 Subject: [PATCH 04/15] add getter/setter policies to provided role because there is no other way it will ever work unless it's granted put/get to all s3 resources --- main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index 4793b1a..d494243 100644 --- a/main.tf +++ b/main.tf @@ -161,18 +161,18 @@ resource "aws_iam_role_policy" "aws_ccm" { } resource "aws_iam_role_policy" "get_token" { - count = var.iam_instance_profile == "" ? 1 : 0 + #count = var.iam_instance_profile == "" ? 1 : 0 name = "${local.uname}-rke2-server-get-token" - role = module.iam[count.index].role + role = var.iam_instance_profile == "" ? module.iam[0].role : data.aws_iam_role.provided[0].name policy = module.statestore.token.policy_document } resource "aws_iam_role_policy" "put_kubeconfig" { - count = var.iam_instance_profile == "" ? 1 : 0 + #count = var.iam_instance_profile == "" ? 1 : 0 name = "${local.uname}-rke2-server-put-kubeconfig" - role = module.iam[count.index].role + role = var.iam_instance_profile == "" ? module.iam[0].role : data.aws_iam_role.provided[0].name policy = module.statestore.kubeconfig_put_policy } From 08a1b56e9509faf9b477669d4f88a61de76eea82 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 5 Dec 2023 09:31:53 -0600 Subject: [PATCH 05/15] optional typing is only non-experimental as of 1.3, so we need to bunch version constraint --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index 03bb5f1..c1662c7 100644 --- a/versions.tf +++ b/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 0.13" + required_version = ">= 1.3" required_providers { aws = { From 04cf7b52a603e17f46effc1f12e71e5673b019c2 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 5 Dec 2023 09:32:40 -0600 Subject: [PATCH 06/15] disable nm-cloud-setup because rke2 will not startup otherwise --- modules/userdata/files/rke2-init.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/userdata/files/rke2-init.sh b/modules/userdata/files/rke2-init.sh index b246b44..61f8014 100644 --- a/modules/userdata/files/rke2-init.sh +++ b/modules/userdata/files/rke2-init.sh @@ -174,6 +174,9 @@ upload() { fi fi + systemctl is-enabled --quiet nm-cloud-setup && \ + systemctl disable nm-cloud-setup; systemctl disable nm-cloud-setup.timer + if [ $TYPE = "server" ]; then # Initialize server identify @@ -189,7 +192,6 @@ upload() { systemctl enable rke2-server systemctl daemon-reload - export KUBECONFIG=/etc/rancher/rke2/rke2.yaml export PATH=$PATH:/var/lib/rancher/rke2/bin From 17aa733f1cdae2e9a2d72d40a5d8ac5801a7a436 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 5 Dec 2023 09:34:29 -0600 Subject: [PATCH 07/15] update vars docs --- README.md | 12 +++++++----- examples/quickstart/main.tf | 2 -- modules/agent-nodepool/README.md | 5 +++-- modules/nodepool/README.md | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6d8247a..f2cc2a8 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,9 @@ Optional policies have the option of being created by default, but are specified | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 0.13 | -| [aws](#requirement\_aws) | ~> 5.4 | -| [cloudinit](#requirement\_cloudinit) | ~> 2.3 | -| [random](#requirement\_random) | ~> 3.5 | +| [aws](#requirement\_aws) | >= 4.6, <= 5.22 | +| [cloudinit](#requirement\_cloudinit) | >= 2 | +| [random](#requirement\_random) | >= 3 | ## Providers @@ -193,6 +193,7 @@ Optional policies have the option of being created by default, but are specified | [random_string.uid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_iam_policy_document.aws_ccm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.aws_required](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.provided](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [cloudinit_config.this](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | ## Inputs @@ -200,7 +201,7 @@ Optional policies have the option of being created by default, but are specified | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Server pool ami | `string` | n/a | yes | -| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `false` | no | +| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `null` | no | | [awscli\_url](#input\_awscli\_url) | URL for awscli zip file | `string` | `"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Server pool block device mapping configuration | `map(string)` | 
{
  "encrypted": false,
  "size": 30
}
| no | | [ccm\_external](#input\_ccm\_external) | Set kubelet arg 'cloud-provider-name' value to 'external'. Requires manual install of CCM. | `bool` | `false` | no | @@ -221,10 +222,11 @@ Optional policies have the option of being created by default, but are specified | [metadata\_options](#input\_metadata\_options) | Instance Metadata Options | `map(any)` | 
{
  "http_endpoint": "enabled",
  "http_put_response_hop_limit": 2,
  "http_tokens": "required",
  "instance_metadata_tags": "disabled"
}
| no | | [post\_userdata](#input\_post\_userdata) | Custom userdata to run immediately after rke2 node attempts to join cluster | `string` | `""` | no | | [pre\_userdata](#input\_pre\_userdata) | Custom userdata to run immediately before rke2 node attempts to join cluster, after required rke2, dependencies are installed | `string` | `""` | no | +| [rke2\_channel](#input\_rke2\_channel) | Channel to use for RKE2 server nodepool | `string` | `null` | no | | [rke2\_config](#input\_rke2\_config) | Server pool additional configuration passed as rke2 config file, see https://docs.rke2.io/install/install_options/server_config for full list of options | `string` | `""` | no | | [rke2\_install\_script\_url](#input\_rke2\_install\_script\_url) | URL for RKE2 install script | `string` | `"https://get.rke2.io"` | no | | [rke2\_start](#input\_rke2\_start) | Start/Stop value for the rke2-server/agent service. This will prevent the service from starting until the next reboot. True=start, False= don't start. | `bool` | `true` | no | -| [rke2\_version](#input\_rke2\_version) | Version to use for RKE2 server nodes | `string` | `"v1.19.7+rke2r1"` | no | +| [rke2\_version](#input\_rke2\_version) | Version to use for RKE2 server nodepool | `string` | `null` | no | | [servers](#input\_servers) | Number of servers to create | `number` | `3` | no | | [spot](#input\_spot) | Toggle spot requests for server pool | `bool` | `false` | no | | [ssh\_authorized\_keys](#input\_ssh\_authorized\_keys) | Server pool list of public keys to add as authorized ssh keys | `list(string)` | `[]` | no | diff --git a/examples/quickstart/main.tf b/examples/quickstart/main.tf index 462b16a..148e03c 100644 --- a/examples/quickstart/main.tf +++ b/examples/quickstart/main.tf @@ -63,7 +63,6 @@ module "rke2" { iam_instance_profile = local.server_iam_role controlplane_internal = false # Note this defaults to best practice of true, but is explicitly set to public for demo purposes tags = local.tags - } # @@ -78,7 +77,6 @@ module "agents" { ssh_authorized_keys = [tls_private_key.ssh.public_key_openssh] tags = local.tags cluster_data = module.rke2.cluster_data - } # For demonstration only, lock down ssh access in production diff --git a/modules/agent-nodepool/README.md b/modules/agent-nodepool/README.md index b76ac24..08d78c0 100644 --- a/modules/agent-nodepool/README.md +++ b/modules/agent-nodepool/README.md @@ -36,7 +36,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | Node pool ami | `string` | `""` | no | -| [asg](#input\_asg) | Node pool AutoScalingGroup scaling definition | 
object({
    min                  = number
    max                  = number
    desired              = number
    suspended_processes  = list(string)
    termination_policies = list(string)
  })
| 
{
  "desired": 1,
  "max": 10,
  "min": 1,
  "suspended_processes": [],
  "termination_policies": [
    "Default"
  ]
}
| no | +| [asg](#input\_asg) | Node pool AutoScalingGroup scaling definition | 
object({
    min                  = number
    max                  = number
    desired              = number
    suspended_processes  = optional(list(string))
    termination_policies = optional(list(string))
  })
| 
{
  "desired": 1,
  "max": 10,
  "min": 1,
  "suspended_processes": [],
  "termination_policies": [
    "Default"
  ]
}
| no | | [awscli\_url](#input\_awscli\_url) | URL for awscli zip file | `string` | `"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip"` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | Node pool block device mapping configuration | `map(string)` | 
{
  "size": 30,
  "type": "gp2"
}
| no | | [ccm\_external](#input\_ccm\_external) | Set kubelet arg 'cloud-provider-name' value to 'external'. Requires manual install of CCM. | `bool` | `false` | no | @@ -54,10 +54,11 @@ | [name](#input\_name) | Nodepool name | `string` | n/a | yes | | [post\_userdata](#input\_post\_userdata) | Custom userdata to run immediately after rke2 node attempts to join cluster | `string` | `""` | no | | [pre\_userdata](#input\_pre\_userdata) | Custom userdata to run immediately before rke2 node attempts to join cluster, after required rke2, dependencies are installed | `string` | `""` | no | +| [rke2\_channel](#input\_rke2\_channel) | Channel to use for RKE2 agent nodepool | `string` | `null` | no | | [rke2\_config](#input\_rke2\_config) | Node pool additional configuration passed as rke2 config file, see https://docs.rke2.io/install/install_options/agent_config for full list of options | `string` | `""` | no | | [rke2\_install\_script\_url](#input\_rke2\_install\_script\_url) | URL for RKE2 install script | `string` | `"https://get.rke2.io"` | no | | [rke2\_start](#input\_rke2\_start) | Start/Stop value for the rke2-server/agent service. True=start, False= don't start. | `bool` | `true` | no | -| [rke2\_version](#input\_rke2\_version) | Version to use for RKE2 server nodepool | `string` | `"v1.19.7+rke2r1"` | no | +| [rke2\_version](#input\_rke2\_version) | Version to use for RKE2 agent nodepool | `string` | `null` | no | | [spot](#input\_spot) | Toggle spot requests for node pool | `bool` | `false` | no | | [ssh\_authorized\_keys](#input\_ssh\_authorized\_keys) | Node pool list of public keys to add as authorized ssh keys, not required | `list(string)` | `[]` | no | | [subnets](#input\_subnets) | List of subnet IDs to create resources in | `list(string)` | n/a | yes | diff --git a/modules/nodepool/README.md b/modules/nodepool/README.md index d77e733..111254d 100644 --- a/modules/nodepool/README.md +++ b/modules/nodepool/README.md @@ -27,7 +27,7 @@ No modules. |------|-------------|------|---------|:--------:| | [ami](#input\_ami) | n/a | `string` | `""` | no | | [asg](#input\_asg) | n/a | 
object({
    min                  = number
    max                  = number
    desired              = number
    suspended_processes  = list(string)
    termination_policies = list(string)
  })
| 
{
  "desired": 3,
  "max": 7,
  "min": 1,
  "suspended_processes": [],
  "termination_policies": []
}
| no | -| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `false` | no | +| [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | n/a | `bool` | `null` | no | | [block\_device\_mappings](#input\_block\_device\_mappings) | n/a | `map(string)` | 
{
  "size": 30,
  "type": "gp2"
}
| no | | [extra\_block\_device\_mappings](#input\_extra\_block\_device\_mappings) | n/a | `list(map(string))` | `[]` | no | | [extra\_cloud\_config\_config](#input\_extra\_cloud\_config\_config) | extra config to append to cloud-config | `string` | `""` | no | From 6bc68eae787cc903329646e8a94e03218b92c7fc Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Sat, 16 Dec 2023 05:37:13 -0600 Subject: [PATCH 08/15] add policies for autoscaler --- README.md | 5 ++++- data.tf | 30 +++++++++++++++++++++++++++ examples/cloud-enabled/README.md | 32 +++++++++++++++++++++++++++-- examples/cloud-enabled/main.tf | 9 ++++++-- main.tf | 8 ++++++++ modules/agent-nodepool/data.tf | 15 ++++++++++++-- modules/userdata/files/rke2-init.sh | 8 ++------ variables.tf | 6 ++++++ 8 files changed, 100 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f2cc2a8..993c7e7 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Optional policies have the option of being created by default, but are specified | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | +| [terraform](#requirement\_terraform) | >= 1.3 | | [aws](#requirement\_aws) | >= 4.6, <= 5.22 | | [cloudinit](#requirement\_cloudinit) | >= 2 | | [random](#requirement\_random) | >= 3 | @@ -179,6 +179,7 @@ Optional policies have the option of being created by default, but are specified | Name | Type | |------|------| +| [aws_iam_role_policy.aws_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.aws_ccm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.aws_required](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.get_token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | @@ -191,6 +192,7 @@ Optional policies have the option of being created by default, but are specified | [aws_security_group_rule.server_cp_supervisor](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [random_password.token](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [random_string.uid](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_iam_policy_document.aws_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.aws_ccm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.aws_required](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_role.provided](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | @@ -212,6 +214,7 @@ Optional policies have the option of being created by default, but are specified | [controlplane\_internal](#input\_controlplane\_internal) | Toggle between public or private control plane load balancer | `bool` | `true` | no | | [create\_acl](#input\_create\_acl) | Toggle creation of ACL for statestore bucket | `bool` | `true` | no | | [download](#input\_download) | Toggle best effort download of rke2 dependencies (rke2 and aws cli), if disabled, dependencies are assumed to exist in $PATH | `bool` | `true` | no | +| [enable\_autoscaler](#input\_enable\_autoscaler) | Toggle enabling policies required for cluster autoscaler to work | `bool` | `false` | no | | [enable\_ccm](#input\_enable\_ccm) | Toggle enabling the cluster as aws aware, this will ensure the appropriate IAM policies are present | `bool` | `false` | no | | [extra\_block\_device\_mappings](#input\_extra\_block\_device\_mappings) | Used to specify additional block device mapping configurations | `list(map(string))` | `[]` | no | | [extra\_cloud\_config\_config](#input\_extra\_cloud\_config\_config) | extra config to append to cloud-config | `string` | `""` | no | diff --git a/data.tf b/data.tf index b677565..d14da20 100644 --- a/data.tf +++ b/data.tf @@ -148,6 +148,36 @@ data "aws_iam_policy_document" "aws_ccm" { } } +data "aws_iam_policy_document" "aws_autoscaler" { + count = var.enable_autoscaler ? 1 : 0 + + statement { + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplateVersions" + ] + effect = "Allow" + resources = ["*"] + } + + statement { + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:DescribeImages", + "ec2:GetInstanceTypesFromInstanceRequirements", + "eks:DescribeNodegroup" + ] + effect = "Allow" + resources = ["*"] + } +} + # Need to add getter/setter for statestore if provided with role data "aws_iam_role" "provided" { count = var.iam_instance_profile == "" ? 0 : 1 diff --git a/examples/cloud-enabled/README.md b/examples/cloud-enabled/README.md index 2d172fa..8ea0329 100644 --- a/examples/cloud-enabled/README.md +++ b/examples/cloud-enabled/README.md @@ -1,9 +1,37 @@ # AWS Cloud Enabled RKE2 -This example demonstrates configuring `rke2` with the in tree AWS Cloud Controller Manager (CCM). +This example demonstrates configuring `rke2` with the Kubernetes `cloud-provider-aws`. -The `rke2` configuration file option is used to configure `cloud-provider-name: aws`. +The in-tree provider is now deprecated and removed completely as of Kubernetes v1.27. The flag to the kubelet to provide a cloud-provider name at all has also been deprecated since v1.24 and is subject to removal at any time. Thus, we will show how to install the out-of-tree provider and autoscaler instead. + +Now, however, that until [cloud-provider-aws/746](https://github.com/kubernetes/cloud-provider-aws/issues/746) is fixed, we need to continue passing `cloud-provider-name: external` to `rke2`, which will still work as of Kubernetes v1.28. Note that resource tagging is extremely important when enabling the AWS CCM, as it is how the controller identifies resources for things such as autoprovisioning load balancers. Regardless of the aws provider being explicitly declared, all resources created by `rke2-aws-tf` are tagged appropriately. However, there are resources not owned by `rke2-aws-tf` proper, such as VPC's and Subnets. This example explicitly declares VPCs and Subnets with the appropriate tags so you can get a feel for all the required resources in your own environment. + +Also see [kubernetes/website/42770](https://github.com/kubernetes/website/issues/42770). We need to tell the Kubernetes Controller Manager here not to try and configure node CIDRs and cloud routes because Calico does not need these. All of the supported CNIs for `rke2` should be using vxlan or something equivalent, not setting routes in the VPC route table directly. + +## `cloud-provider-aws` + +```sh +helm repo add aws-cloud-controller-manager https://kubernetes.github.io/cloud-provider-aws +helm repo update +helm install aws-cloud-controller-manager aws-cloud-controller-manager/aws-cloud-controller-manager \ + --namespace kube-system \ + --set-json 'args=["--v=2", "--cloud-provider=aws", "--allocate-node-cidrs=false", "--configure-cloud-routes=false"]' \ + --set-string 'nodeSelector.node-role\.kubernetes\.io/control-plane=true' +``` + +## `cluster-autoscaler` + +Match region to your actual region, but it is `us-gov-west-1` in this example. + +```sh +helm repo add autoscaler https://kubernetes.github.io/autoscaler +helm repo update +helm install autoscaler autoscaler/cluster-autoscaler \ + --namespace kube-system \ + --set autoDiscovery.clusterName=cloud-enabled-zjl \ + --set awsRegion=us-gov-west-1 +``` diff --git a/examples/cloud-enabled/main.tf b/examples/cloud-enabled/main.tf index 8ac4bce..2bac0bf 100644 --- a/examples/cloud-enabled/main.tf +++ b/examples/cloud-enabled/main.tf @@ -96,16 +96,19 @@ module "rke2" { vpc_id = module.vpc.vpc_id subnets = module.vpc.public_subnets # Note: Public subnets used for demo purposes, this is not recommended in production - ami = data.aws_ami.rhel8.image_id # Note: Multi OS is primarily for example purposes + ami = data.aws_ami.rhel8.image_id ssh_authorized_keys = [tls_private_key.ssh.public_key_openssh] instance_type = "t3.medium" controlplane_internal = false # Note this defaults to best practice of true, but is explicitly set to public for demo purposes servers = 1 # Enable AWS Cloud Controller Manager - enable_ccm = true + enable_ccm = true + enable_autoscaler = true rke2_config = yamlencode({ "node-label" : ["name=server", "os=rhel8"] }) + + rke2_channel = "v1.27" } # @@ -131,6 +134,8 @@ module "agents" { rke2_config = yamlencode({ "node-label" : ["name=generic", "os=rhel8"] }) cluster_data = module.rke2.cluster_data + + rke2_channel = "v1.27" } # For demonstration only, lock down ssh access in production diff --git a/main.tf b/main.tf index d494243..b2b5f71 100644 --- a/main.tf +++ b/main.tf @@ -160,6 +160,14 @@ resource "aws_iam_role_policy" "aws_ccm" { policy = data.aws_iam_policy_document.aws_ccm[count.index].json } +resource "aws_iam_role_policy" "aws_autoscaler" { + count = var.iam_instance_profile == "" && var.enable_autoscaler ? 1 : 0 + + name = "${local.uname}-rke2-server-aws-autoscaler" + role = module.iam[count.index].role + policy = data.aws_iam_policy_document.aws_autoscaler[count.index].json +} + resource "aws_iam_role_policy" "get_token" { #count = var.iam_instance_profile == "" ? 1 : 0 diff --git a/modules/agent-nodepool/data.tf b/modules/agent-nodepool/data.tf index 71db84d..dfb5a2f 100644 --- a/modules/agent-nodepool/data.tf +++ b/modules/agent-nodepool/data.tf @@ -34,11 +34,22 @@ data "aws_iam_policy_document" "aws_autoscaler" { "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", "autoscaling:DescribeTags", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplateVersions" + ] + } + + statement { + effect = "Allow" + resources = ["*"] + actions = [ "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup", - "ec2:DescribeLaunchTemplateVersions", - "ec2:DescribeInstanceTypes" + "ec2:DescribeImages", + "ec2:GetInstanceTypesFromInstanceRequirements", + "eks:DescribeNodegroup" ] } } diff --git a/modules/userdata/files/rke2-init.sh b/modules/userdata/files/rke2-init.sh index 61f8014..299295d 100644 --- a/modules/userdata/files/rke2-init.sh +++ b/modules/userdata/files/rke2-init.sh @@ -166,12 +166,8 @@ upload() { fetch_token if [ $CCM = "true" ]; then - if [ $CCM_EXTERNAL = "true" ]; then - append_config 'cloud-provider-name: "external"' - append_config 'disable-cloud-controller: "true"' - else - append_config 'cloud-provider-name: "aws"' - fi + append_config 'cloud-provider-name: "external"' + append_config 'disable-cloud-controller: "true"' fi systemctl is-enabled --quiet nm-cloud-setup && \ diff --git a/variables.tf b/variables.tf index a72152e..983ecba 100644 --- a/variables.tf +++ b/variables.tf @@ -180,6 +180,12 @@ variable "post_userdata" { default = "" } +variable "enable_autoscaler" { + description = "Toggle enabling policies required for cluster autoscaler to work" + type = bool + default = false +} + variable "enable_ccm" { description = "Toggle enabling the cluster as aws aware, this will ensure the appropriate IAM policies are present" type = bool From 9849d3f890c60c9c88a29b31272a14c464c084ef Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Mon, 18 Dec 2023 13:59:14 -0600 Subject: [PATCH 09/15] allow separate subnets for lb and cluster nodes and attach lb sg to lb --- main.tf | 8 ++++++-- modules/nlb/main.tf | 1 + variables.tf | 8 +++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index b2b5f71..9246f8f 100644 --- a/main.tf +++ b/main.tf @@ -16,6 +16,8 @@ locals { cluster_sg = aws_security_group.cluster.id token = module.statestore.token } + + lb_subnets = var.lb_subnets == null ? var.subnets : var.lb_subnets target_group_arns = module.cp_lb.target_group_arns } @@ -53,7 +55,7 @@ module "cp_lb" { source = "./modules/nlb" name = local.uname vpc_id = var.vpc_id - subnets = var.subnets + subnets = local.lb_subnets enable_cross_zone_load_balancing = var.controlplane_enable_cross_zone_load_balancing internal = var.controlplane_internal @@ -197,7 +199,9 @@ module "servers" { instance_type = var.instance_type block_device_mappings = var.block_device_mappings extra_block_device_mappings = var.extra_block_device_mappings - vpc_security_group_ids = concat([aws_security_group.server.id, aws_security_group.cluster.id, module.cp_lb.security_group], var.extra_security_group_ids) + vpc_security_group_ids = concat( + [aws_security_group.cluster.id, aws_security_group.server.id], + var.extra_security_group_ids) spot = var.spot target_group_arns = local.target_group_arns wait_for_capacity_timeout = var.wait_for_capacity_timeout diff --git a/modules/nlb/main.tf b/modules/nlb/main.tf index 21187d3..4642925 100644 --- a/modules/nlb/main.tf +++ b/modules/nlb/main.tf @@ -85,6 +85,7 @@ resource "aws_lb" "controlplane" { internal = var.internal load_balancer_type = "network" subnets = var.subnets + security_groups = [aws_security_group.controlplane.id] enable_cross_zone_load_balancing = var.enable_cross_zone_load_balancing diff --git a/variables.tf b/variables.tf index 983ecba..135c9c7 100644 --- a/variables.tf +++ b/variables.tf @@ -14,8 +14,14 @@ variable "vpc_id" { type = string } +variable "lb_subnets" { + description = "List of subnet IDs to create load balancer in" + default = null + type = list(string) +} + variable "subnets" { - description = "List of subnet IDs to create resources in" + description = "List of subnet IDs to create nodes in" type = list(string) } From e1e07ed6530fd383ab25209eaff3f0fee2668040 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Mon, 18 Dec 2023 14:08:36 -0600 Subject: [PATCH 10/15] update docs --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 993c7e7..6faf799 100644 --- a/README.md +++ b/README.md @@ -161,9 +161,9 @@ Optional policies have the option of being created by default, but are specified | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.11.0 | -| [cloudinit](#provider\_cloudinit) | 2.3.2 | -| [random](#provider\_random) | 3.5.1 | +| [aws](#provider\_aws) | >= 4.6, <= 5.22 | +| [cloudinit](#provider\_cloudinit) | >= 2 | +| [random](#provider\_random) | >= 3 | ## Modules @@ -222,6 +222,7 @@ Optional policies have the option of being created by default, but are specified | [iam\_instance\_profile](#input\_iam\_instance\_profile) | Server pool IAM Instance Profile, created if left blank (default behavior) | `string` | `""` | no | | [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | If provided, the IAM role created for the servers will be created with this permissions boundary attached. | `string` | `null` | no | | [instance\_type](#input\_instance\_type) | Server pool instance type | `string` | `"t3a.medium"` | no | +| [lb\_subnets](#input\_lb\_subnets) | List of subnet IDs to create load balancer in | `list(string)` | `null` | no | | [metadata\_options](#input\_metadata\_options) | Instance Metadata Options | `map(any)` | 
{
  "http_endpoint": "enabled",
  "http_put_response_hop_limit": 2,
  "http_tokens": "required",
  "instance_metadata_tags": "disabled"
}
| no | | [post\_userdata](#input\_post\_userdata) | Custom userdata to run immediately after rke2 node attempts to join cluster | `string` | `""` | no | | [pre\_userdata](#input\_pre\_userdata) | Custom userdata to run immediately before rke2 node attempts to join cluster, after required rke2, dependencies are installed | `string` | `""` | no | @@ -234,7 +235,7 @@ Optional policies have the option of being created by default, but are specified | [spot](#input\_spot) | Toggle spot requests for server pool | `bool` | `false` | no | | [ssh\_authorized\_keys](#input\_ssh\_authorized\_keys) | Server pool list of public keys to add as authorized ssh keys | `list(string)` | `[]` | no | | [statestore\_attach\_deny\_insecure\_transport\_policy](#input\_statestore\_attach\_deny\_insecure\_transport\_policy) | Toggle for enabling s3 policy to reject non-SSL requests | `bool` | `true` | no | -| [subnets](#input\_subnets) | List of subnet IDs to create resources in | `list(string)` | n/a | yes | +| [subnets](#input\_subnets) | List of subnet IDs to create nodes in | `list(string)` | n/a | yes | | [suspended\_processes](#input\_suspended\_processes) | List of processes to suspend in the autoscaling service | `list(string)` | `[]` | no | | [tags](#input\_tags) | Map of tags to add to all resources created | `map(string)` | `{}` | no | | [termination\_policies](#input\_termination\_policies) | List of policies to decide how the instances in the Auto Scaling Group should be terminated | `list(string)` | 
[
  "Default"
]
| no | From 26629708e56f3d961cfefaa19052e93d958f4469 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Mon, 18 Dec 2023 14:11:04 -0600 Subject: [PATCH 11/15] terraform fmt --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9246f8f..3c19b1b 100644 --- a/main.tf +++ b/main.tf @@ -17,7 +17,7 @@ locals { token = module.statestore.token } - lb_subnets = var.lb_subnets == null ? var.subnets : var.lb_subnets + lb_subnets = var.lb_subnets == null ? var.subnets : var.lb_subnets target_group_arns = module.cp_lb.target_group_arns } From c830bc2fe8bfff2748b3ac3090d07f16651a8906 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 19 Dec 2023 06:42:12 -0600 Subject: [PATCH 12/15] add support for rhel9 --- modules/common/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/common/download.sh b/modules/common/download.sh index d4dc962..fc971a7 100644 --- a/modules/common/download.sh +++ b/modules/common/download.sh @@ -86,7 +86,7 @@ do_download() { INSTALL_RKE2_METHOD='yum' INSTALL_RKE2_TYPE="${type}" ./install.sh ;; - 8*) + [8-9]*) INSTALL_RKE2_METHOD='yum' INSTALL_RKE2_TYPE="${type}" ./install.sh ;; From 6b99eece013977d1030727c33d8bac8ddad07320 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 19 Dec 2023 06:43:36 -0600 Subject: [PATCH 13/15] add example for cluster in private subnets fronted by public lb --- examples/public-lb/README.md | 31 +++++++ examples/public-lb/main.tf | 161 +++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 examples/public-lb/README.md create mode 100644 examples/public-lb/main.tf diff --git a/examples/public-lb/README.md b/examples/public-lb/README.md new file mode 100644 index 0000000..3a050e0 --- /dev/null +++ b/examples/public-lb/README.md @@ -0,0 +1,31 @@ +# public-lb + +Demonstrates a cluster with nodes spread across three availability zones in private subnets with a load balancer for the API server in the same availability zones but public subnets. It is mandatory in AWS to put an Internet-facing load balancer in public subnets or it will not be routable from the Internet, in spite of having public IPs. + +## Connecting to cluster nodes + +Running the following as-is requires `jq` and `sshuttle`, but it is also possible to do this the hard way if you're familiar with ssh tunneling. + +Get the IP address of the bastion, the generated cluster name, and the VPC CIDR range: + +```sh +BASTION_IP=$(terraform output -raw bastion_ip) +CLUSTER_NAME=$(terraform output -json cluster_data | jq '.cluster_name' | tr -d '"') +VPC_CIDR=$(terraform output -raw vpc_cidr) +``` + +Establish an ssh tunnel using `sshuttle`: + +```sh +sshuttle -r ec2-user@$BASTION_IP $VPC_CIDR --ssh-cmd "ssh -i $CLUSTER_NAME.pem" +``` + +In a second terminal, get the private IP of the first cluster node. We're just picking one here to demonstrate, but of course you can choose any other or all at once if you want to use `tmux` to establish simulataneous connections from the same terminal. The following assume you're still in the same directory Terraform was run from. + +```sh +NODE0_IP=$(aws ec2 describe-instances \ + --filter "Name=tag:aws:autoscaling:groupName,Values=public-lb*" \ + --query "Reservations[*].Instances[0].PrivateIpAddress" \ + --output text) +ssh -i $CLUSTER_NAME.pem ec2-user@$NODE0_IP +``` diff --git a/examples/public-lb/main.tf b/examples/public-lb/main.tf new file mode 100644 index 0000000..26cec6e --- /dev/null +++ b/examples/public-lb/main.tf @@ -0,0 +1,161 @@ +locals { + cidr = "10.0.0.0/16" + cluster_name = "public-lb" + rke2_instance_type = "t3.medium" + rke2_channel = "stable" + servers = 3 + + tags = { + "terraform" = "true", + "env" = "public-lb", + } +} + +provider "aws" { + default_tags { + tags = local.tags + } +} + +data "aws_ami" "rhel9" { + most_recent = true + owners = ["219670896067"] # owner is specific to aws gov cloud + + filter { + name = "name" + values = ["RHEL-9*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } +} + +data "aws_region" "current" {} + +data "cloudinit_config" "bastion" { + gzip = true + base64_encode = true + + part { + filename = "00_init.sh" + content_type = "text/x-shellscript" + # required to use sshuttle for ssh tunneling + content = < Date: Tue, 19 Dec 2023 06:47:09 -0600 Subject: [PATCH 14/15] add a README for examples --- examples/README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 examples/README.md diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..d618436 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,48 @@ +# examples + +[cloud-enabled](./cloud-enabled/): Shows a cluster with nodepools having the appropriate IAM policies and tags to enable using `cloud-provider-aws`, `cluster-autoscaler`, and `ebs-csi-driver` with KMS volume encryption. + +[public-lb](./public-lb/): Shows a cluster with nodepools in private VPC subnets with the API server fronted by an Internet-facing load balancer. + +[quickstart](./quickstart/): Minimal deployment with automated kubeconfig fetching and ssh to cluster nodes allowed from anywhere on the Internet. + +## AMI queries + +The Terraform data resource `aws_ami` constructs an API query such that the block of hcl below: + +```hcl +data "aws_ami" "rhel9" { + most_recent = true + owners = ["219670896067"] + + filter { + name = "name" + values = ["RHEL-9*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } +} +``` + +Is equivalent to the AWS CLI call: + +```sh +aws ec2 describe-images \ + --owners 219670896067 \ + --filter "Name=name,Values=RHEL-9*" \ + --filter "Name=architecture,Values=x86_64" \ + --query "reverse(sort_by(Images, &CreationDate))[0]" +``` + +## Interacting with your cluster + +The `cloud-init` user-data scripts provided with this module always upload the default admin kubeconfig from `/etc/rancher/rke2/rke2.yaml` to the S3 statestore bucket upon successful startup of the `rke2-server` service on the first cluster node, editing the localhost server URL to use the DNS name of the load balancer. Some examples demonstrate a Terraform null resource to download this. To download using AWS CLI and then use it: + +```sh +aws s3 cp s3://public-lb-gkw-rke2/rke2.yaml . +chmod 0600 rke2.yaml +export KUBECONFIG="$(pwd)/rke2.yaml" +``` From 1020de30f63760665f915758e5a6c62ac3c4f7a2 Mon Sep 17 00:00:00 2001 From: Adam Acosta Date: Tue, 19 Dec 2023 06:57:14 -0600 Subject: [PATCH 15/15] format fix --- examples/public-lb/main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/public-lb/main.tf b/examples/public-lb/main.tf index 26cec6e..6f4cc84 100644 --- a/examples/public-lb/main.tf +++ b/examples/public-lb/main.tf @@ -42,7 +42,7 @@ data "cloudinit_config" "bastion" { filename = "00_init.sh" content_type = "text/x-shellscript" # required to use sshuttle for ssh tunneling - content = <