feat: use Amazon Linux 2023 for runner manager instance (#1138)

## Description

Adds support for Amazon Linux 2023 and removes the `dmesg` logs from
Cloudwatch as it wasn't useful at all.

Closes #1137

- [x] GitlabRunner is operating
- [x] Cloudwatch logs are populated
- [x] jobs form GitLab are processed
- [x] no errors in log files found
- [x] also works with the old default AMI (Amazon Linux 1 and 2)

---------

Co-authored-by: Matthias Kay <github@matthiaskay.de>

template/logging.tftpl

@@ -3,48 +3,53 @@ echo 'installing additional software for logging'
 for i in {1..7}
 do
   echo "Attempt: ---- " $i
-  yum install -y aws-cli awslogs jq && break || sleep 60
+  yum install -y aws-cli rsyslog amazon-cloudwatch-agent jq && break || sleep 60
 done
 
-# Inject the CloudWatch Logs configuration file contents
-cat > /etc/awslogs/awslogs.conf <<- EOF
-[general]
-state_file = /var/lib/awslogs/agent-state
-
-[/var/log/dmesg]
-file = /var/log/dmesg
-log_stream_name = {instanceId}/dmesg
-log_group_name = ${log_group_name}
-initial_position = start_of_file
-
-[/var/log/messages]
-file = /var/log/messages
-log_stream_name = {instanceId}/messages
-log_group_name = ${log_group_name}
-datetime_format = %b %d %H:%M:%S
-initial_position = start_of_file
-
-[/var/log/user-data.log]
-file = /var/log/user-data.log
-log_stream_name = {instanceId}/user-data
-log_group_name = ${log_group_name}
-initial_position = start_of_file
+# Create the CloudWatch Logs configuration file
+cat > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json  <<- EOF 
+{
+    "agent": {
+        "run_as_user": "root"
+    },
+    "metrics": {
+        "metrics_collected": {
+            "cpu": {
+                "measurement": [
+                    "cpu_usage_idle"
+                ],
+                "metrics_collection_interval": 300,
+                "totalcpu": true
+            },
+            "mem": {
+                "measurement": [
+                    "mem_used_percent"
+                ],
+                "metrics_collection_interval": 300
+            }
+        }
+    },
+    "logs": {
+        "logs_collected": {
+            "files": {
+                "collect_list": [
+                    {
+                        "file_path": "/var/log/messages",
+                        "log_group_name": "${log_group_name}",
+                        "log_stream_name": "{instance_id}/messages"
+                    },
+                    {
+                        "file_path": "/var/log/user-data.log",
+                        "log_group_name": "${log_group_name}",
+                        "log_stream_name": "{instance_id}/user-data"
+                    }
+                ]
+            }
+        }
+    }
+}
 
 EOF
 
-# Set the region to send CloudWatch Logs data to (the region where the instance is located)
-region=$(curl -s -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
-sed -i -e "s/region = us-east-1/region = $region/g" /etc/awslogs/awscli.conf
-
-# Replace instance id.
-instanceId=$(curl -s -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .instanceId)
-sed -i -e "s/{instanceId}/$instanceId/g" /etc/awslogs/awslogs.conf
-
-if grep -q ':2$' /etc/system-release-cpe  ; then
-  # AWS Linux 2 renamed the awslogs service to awslogsd and uses systemd
-  systemctl enable awslogsd
-  systemctl start awslogsd
-else
-  service awslogs start
-  chkconfig awslogs on
-fi
+systemctl --now enable rsyslog
+systemctl restart amazon-cloudwatch-agent

variables.tf

@@ -135,11 +135,11 @@ variable "runner_instance" {
 }
 
 variable "runner_ami_filter" {
-  description = "List of maps used to create the AMI filter for the Runner AMI. Must resolve to an Amazon Linux 1 or 2 image."
+  description = "List of maps used to create the AMI filter for the Runner AMI. Must resolve to an Amazon Linux 1, 2 or 2023 image."
   type        = map(list(string))
 
   default = {
-    name = ["amzn2-ami-hvm-2.*-x86_64-ebs"]
+    name = ["al2023-ami-2023*-x86_64"]
   }
 }