Fix runner registration reliability and add comprehensive logging

env.json

@@ -0,0 +1,11 @@
+{
+  "GitHubActionHookFunction": {
+    "GITHUB_PAT_SECRET_NAME": "github-runner-autoscaler-pat",
+    "EXTRA_RUNNER_LABELS": ",vor_stream_sdlc",
+    "IMAGE_ID": "ami-0c0c88099397fccb4",
+    "SUBNET_ID": "subnet-0123456789def",
+    "SECURITY_GROUP_IDS": "sg-0123456789def",
+    "KEY_NAME": "terraform-2025051802",
+    "INSTANCE_PROFILE_ARN": "arn:aws:iam::123456789012:instance-profile/github-runner-autoscaler-RunnerInstanceProfile-XXXXX"
+  }
+}

event-custom-instance.json

@@ -0,0 +1,55 @@
+{
+  "resource": "/",
+  "path": "/",
+  "httpMethod": "POST",
+  "headers": {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+    "X-GitHub-Event": "workflow_job",
+    "X-GitHub-Delivery": "12345678-1234-1234-1234-123456789012",
+    "X-Hub-Signature-256": "sha256=fake_signature_for_testing"
+  },
+  "multiValueHeaders": {
+    "Accept": ["application/json"],
+    "Content-Type": ["application/json"],
+    "X-GitHub-Event": ["workflow_job"],
+    "X-GitHub-Delivery": ["12345678-1234-1234-1234-123456789012"],
+    "X-Hub-Signature-256": ["sha256=fake_signature_for_testing"]
+  },
+  "queryStringParameters": null,
+  "multiValueQueryStringParameters": null,
+  "pathParameters": null,
+  "stageVariables": null,
+  "requestContext": {
+    "resourceId": "123456",
+    "resourcePath": "/",
+    "httpMethod": "POST",
+    "extendedRequestId": "request-id",
+    "requestTime": "09/Apr/2024:08:00:00 +0000",
+    "path": "/",
+    "accountId": "123456789012",
+    "protocol": "HTTP/1.1",
+    "stage": "Prod",
+    "domainPrefix": "api",
+    "requestTimeEpoch": 1712649600000,
+    "requestId": "12345678-1234-1234-1234-123456789012",
+    "identity": {
+      "cognitoIdentityPoolId": null,
+      "accountId": null,
+      "cognitoIdentityId": null,
+      "caller": null,
+      "sourceIp": "192.0.2.1",
+      "principalOrgId": null,
+      "accessKey": null,
+      "cognitoAuthenticationType": null,
+      "cognitoAuthenticationProvider": null,
+      "userArn": null,
+      "userAgent": "GitHub-Hookshot/abc123",
+      "user": null
+    },
+    "domainName": "api.example.com",
+    "apiId": "1234567890"
+  },
+  "body": "{\"action\":\"queued\",\"workflow_job\":{\"id\":123456789,\"run_id\":987654321,\"workflow_name\":\"CI\",\"head_branch\":\"main\",\"run_url\":\"https://github.com/frgrisk/test-repo/actions/runs/987654321\",\"run_attempt\":1,\"node_id\":\"CR_kwDOABCDEF\",\"head_sha\":\"abcdef0123456789abcdef0123456789abcdef01\",\"url\":\"https://api.github.com/repos/frgrisk/test-repo/actions/jobs/123456789\",\"html_url\":\"https://github.com/frgrisk/test-repo/actions/runs/987654321/job/123456789\",\"status\":\"queued\",\"conclusion\":null,\"created_at\":\"2024-04-09T08:00:00Z\",\"started_at\":null,\"completed_at\":null,\"name\":\"test-job\",\"steps\":[],\"check_run_url\":\"https://api.github.com/repos/frgrisk/test-repo/check-runs/123456789\",\"labels\":[\"self-hosted\",\"ephemeral\",\"X64\",\"Linux\",\"instance-type:m5.large\"],\"runner_id\":null,\"runner_name\":null,\"runner_group_id\":null,\"runner_group_name\":null},\"repository\":{\"id\":123456789,\"node_id\":\"R_kgDOABCDEF\",\"name\":\"test-repo\",\"full_name\":\"frgrisk/test-repo\",\"private\":true,\"owner\":{\"login\":\"frgrisk\",\"id\":12345678,\"node_id\":\"O_kgDOABCDEF\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/12345678?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/frgrisk\",\"html_url\":\"https://github.com/frgrisk\",\"followers_url\":\"https://api.github.com/users/frgrisk/followers\",\"following_url\":\"https://api.github.com/users/frgrisk/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/frgrisk/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/frgrisk/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/frgrisk/subscriptions\",\"organizations_url\":\"https://api.github.com/users/frgrisk/orgs\",\"repos_url\":\"https://api.github.com/users/frgrisk/repos\",\"events_url\":\"https://api.github.com/users/frgrisk/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/frgrisk/received_events\",\"type\":\"Organization\",\"site_admin\":false},\"html_url\":\"https://github.com/frgrisk/test-repo\",\"description\":\"Test repository\",\"fork\":false,\"url\":\"https://api.github.com/repos/frgrisk/test-repo\"},\"organization\":{\"login\":\"frgrisk\",\"id\":12345678,\"node_id\":\"O_kgDOABCDEF\",\"url\":\"https://api.github.com/orgs/frgrisk\",\"repos_url\":\"https://api.github.com/orgs/frgrisk/repos\",\"events_url\":\"https://api.github.com/orgs/frgrisk/events\",\"hooks_url\":\"https://api.github.com/orgs/frgrisk/hooks\",\"issues_url\":\"https://api.github.com/orgs/frgrisk/issues\",\"members_url\":\"https://api.github.com/orgs/frgrisk/members{/member}\",\"public_members_url\":\"https://api.github.com/orgs/frgrisk/public_members{/member}\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/12345678?v=4\",\"description\":\"FRG Risk\"},\"sender\":{\"login\":\"github-actions[bot]\",\"id\":41898282,\"node_id\":\"MDM6Qm90NDE4OTgyODI=\",\"avatar_url\":\"https://avatars.githubusercontent.com/in/15368?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/github-actions%5Bbot%5D\",\"html_url\":\"https://github.com/apps/github-actions\",\"followers_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/followers\",\"following_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/subscriptions\",\"organizations_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/orgs\",\"repos_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/repos\",\"events_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/received_events\",\"type\":\"Bot\",\"site_admin\":false}}",
+  "isBase64Encoded": false
+}

event.json

@@ -0,0 +1,55 @@
+{
+  "resource": "/",
+  "path": "/",
+  "httpMethod": "POST",
+  "headers": {
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+    "X-GitHub-Event": "workflow_job",
+    "X-GitHub-Delivery": "12345678-1234-1234-1234-123456789012",
+    "X-Hub-Signature-256": "sha256=fake_signature_for_testing"
+  },
+  "multiValueHeaders": {
+    "Accept": ["application/json"],
+    "Content-Type": ["application/json"],
+    "X-GitHub-Event": ["workflow_job"],
+    "X-GitHub-Delivery": ["12345678-1234-1234-1234-123456789012"],
+    "X-Hub-Signature-256": ["sha256=fake_signature_for_testing"]
+  },
+  "queryStringParameters": null,
+  "multiValueQueryStringParameters": null,
+  "pathParameters": null,
+  "stageVariables": null,
+  "requestContext": {
+    "resourceId": "123456",
+    "resourcePath": "/",
+    "httpMethod": "POST",
+    "extendedRequestId": "request-id",
+    "requestTime": "09/Apr/2024:08:00:00 +0000",
+    "path": "/",
+    "accountId": "123456789012",
+    "protocol": "HTTP/1.1",
+    "stage": "Prod",
+    "domainPrefix": "api",
+    "requestTimeEpoch": 1712649600000,
+    "requestId": "12345678-1234-1234-1234-123456789012",
+    "identity": {
+      "cognitoIdentityPoolId": null,
+      "accountId": null,
+      "cognitoIdentityId": null,
+      "caller": null,
+      "sourceIp": "192.0.2.1",
+      "principalOrgId": null,
+      "accessKey": null,
+      "cognitoAuthenticationType": null,
+      "cognitoAuthenticationProvider": null,
+      "userArn": null,
+      "userAgent": "GitHub-Hookshot/abc123",
+      "user": null
+    },
+    "domainName": "api.example.com",
+    "apiId": "1234567890"
+  },
+  "body": "{\"action\":\"queued\",\"workflow_job\":{\"id\":123456789,\"run_id\":987654321,\"workflow_name\":\"CI\",\"head_branch\":\"main\",\"run_url\":\"https://github.com/frgrisk/test-repo/actions/runs/987654321\",\"run_attempt\":1,\"node_id\":\"CR_kwDOABCDEF\",\"head_sha\":\"abcdef0123456789abcdef0123456789abcdef01\",\"url\":\"https://api.github.com/repos/frgrisk/test-repo/actions/jobs/123456789\",\"html_url\":\"https://github.com/frgrisk/test-repo/actions/runs/987654321/job/123456789\",\"status\":\"queued\",\"conclusion\":null,\"created_at\":\"2024-04-09T08:00:00Z\",\"started_at\":null,\"completed_at\":null,\"name\":\"test-job\",\"steps\":[],\"check_run_url\":\"https://api.github.com/repos/frgrisk/test-repo/check-runs/123456789\",\"labels\":[\"self-hosted\",\"ephemeral\",\"X64\",\"Linux\"],\"runner_id\":null,\"runner_name\":null,\"runner_group_id\":null,\"runner_group_name\":null},\"repository\":{\"id\":123456789,\"node_id\":\"R_kgDOABCDEF\",\"name\":\"test-repo\",\"full_name\":\"frgrisk/test-repo\",\"private\":true,\"owner\":{\"login\":\"frgrisk\",\"id\":12345678,\"node_id\":\"O_kgDOABCDEF\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/12345678?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/frgrisk\",\"html_url\":\"https://github.com/frgrisk\",\"followers_url\":\"https://api.github.com/users/frgrisk/followers\",\"following_url\":\"https://api.github.com/users/frgrisk/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/frgrisk/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/frgrisk/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/frgrisk/subscriptions\",\"organizations_url\":\"https://api.github.com/users/frgrisk/orgs\",\"repos_url\":\"https://api.github.com/users/frgrisk/repos\",\"events_url\":\"https://api.github.com/users/frgrisk/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/frgrisk/received_events\",\"type\":\"Organization\",\"site_admin\":false},\"html_url\":\"https://github.com/frgrisk/test-repo\",\"description\":\"Test repository\",\"fork\":false,\"url\":\"https://api.github.com/repos/frgrisk/test-repo\"},\"organization\":{\"login\":\"frgrisk\",\"id\":12345678,\"node_id\":\"O_kgDOABCDEF\",\"url\":\"https://api.github.com/orgs/frgrisk\",\"repos_url\":\"https://api.github.com/orgs/frgrisk/repos\",\"events_url\":\"https://api.github.com/orgs/frgrisk/events\",\"hooks_url\":\"https://api.github.com/orgs/frgrisk/hooks\",\"issues_url\":\"https://api.github.com/orgs/frgrisk/issues\",\"members_url\":\"https://api.github.com/orgs/frgrisk/members{/member}\",\"public_members_url\":\"https://api.github.com/orgs/frgrisk/public_members{/member}\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/12345678?v=4\",\"description\":\"FRG Risk\"},\"sender\":{\"login\":\"github-actions[bot]\",\"id\":41898282,\"node_id\":\"MDM6Qm90NDE4OTgyODI=\",\"avatar_url\":\"https://avatars.githubusercontent.com/in/15368?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/github-actions%5Bbot%5D\",\"html_url\":\"https://github.com/apps/github-actions\",\"followers_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/followers\",\"following_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/subscriptions\",\"organizations_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/orgs\",\"repos_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/repos\",\"events_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/github-actions%5Bbot%5D/received_events\",\"type\":\"Bot\",\"site_admin\":false}}",
+  "isBase64Encoded": false
+}

main.go

@@ -79,7 +79,7 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 
 		secretOut, err := sm.GetSecretValue(context.TODO(), &secretsmanager.GetSecretValueInput{SecretId: aws.String(secretName)})
 		if err != nil {
-			slog.Error("failed to get secret", "error", err.Error())
+			slog.Error("failed to get secret", "secret", secretName, "error", err.Error())
 
 			return events.APIGatewayProxyResponse{StatusCode: http.StatusInternalServerError}, err
 		}
@@ -114,6 +114,13 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 			return events.APIGatewayProxyResponse{StatusCode: http.StatusInternalServerError}, errors.New("key name missing")
 		}
 
+		instanceProfileArn := os.Getenv("INSTANCE_PROFILE_ARN")
+		if instanceProfileArn == "" {
+			slog.Error("INSTANCE_PROFILE_ARN env var not set")
+
+			return events.APIGatewayProxyResponse{StatusCode: http.StatusInternalServerError}, errors.New("instance profile arn missing")
+		}
+
 		imageID := os.Getenv("IMAGE_ID")
 		if imageID == "" {
 			slog.Error("IMAGE_ID env var not set")
@@ -175,6 +182,9 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 				ImageId:                           aws.String(imageID),
 				InstanceInitiatedShutdownBehavior: types.ShutdownBehaviorTerminate,
 				InstanceType:                      instanceType,
+				IamInstanceProfile: &types.IamInstanceProfileSpecification{
+					Arn: aws.String(instanceProfileArn),
+				},
 				NetworkInterfaces: []types.InstanceNetworkInterfaceSpecification{
 					{
 						AssociatePublicIpAddress: aws.Bool(true),

template.yaml

@@ -29,6 +29,37 @@ Globals:
     MemorySize: 128
 
 Resources:
+  # IAM Role for EC2 instances
+  RunnerInstanceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: ec2.amazonaws.com
+            Action: 'sts:AssumeRole'
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
+      Policies:
+        - PolicyName: CloudWatchLogsPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                  - logs:DescribeLogStreams
+                Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/ec2/github-runner:*'
+
+  RunnerInstanceProfile:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      Roles:
+        - !Ref RunnerInstanceRole
   GitHubActionHookFunction:
     Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
     Properties:
@@ -54,6 +85,7 @@ Resources:
           SUBNET_ID: !Ref SubnetId
           SECURITY_GROUP_IDS: !Ref SecurityGroupIds
           KEY_NAME: !Ref KeyName
+          INSTANCE_PROFILE_ARN: !GetAtt RunnerInstanceProfile.Arn
       Policies:
         - Statement:
             - Sid: RunInstances