Kraken timeout: gossip-router pods not recognized after restart

[kami619]

Describe the bug

There seems to be an issue with Kraken recognizing the pods which are coming back for gossip-router and fails with a timeout message. Need to debug this, at least some form of regression from the past.

[chaos:kraken-pod-scenarios] 2024-08-15 14:26:54,129 [INFO] Executing scenarios for iteration 0
[chaos:kraken-pod-scenarios] 2024-08-15 14:26:54,144 [INFO] scenario scenarios/pod_scenario.yaml
[chaos:kraken-pod-scenarios] 2024-08-15 14:26:55,532 [INFO] {
[chaos:kraken-pod-scenarios] 	"output_id": "success",
[chaos:kraken-pod-scenarios] 	"output_data": {
[chaos:kraken-pod-scenarios] 		"pods": {
[chaos:kraken-pod-scenarios] 			"1723732014361533777": {
[chaos:kraken-pod-scenarios] 				"namespace": "runner-keycloak",
[chaos:kraken-pod-scenarios] 				"name": "infinispan-router-766c85c469-qd97c",
[chaos:kraken-pod-scenarios] 				"creation_timestamp": "2024-08-15 13:17:01+00:00"
[chaos:kraken-pod-scenarios] 			}
[chaos:kraken-pod-scenarios] 		}
[chaos:kraken-pod-scenarios] 	}
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,642 [INFO] {
[chaos:kraken-pod-scenarios] 	"output_id": "error",
[chaos:kraken-pod-scenarios] 	"output_data": {
[chaos:kraken-pod-scenarios] 		"error": "timeout while waiting for pods to come up"
[chaos:kraken-pod-scenarios] 	}
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,643 [ERROR] scenario: scenarios/pod_scenario.yaml failed with exception: <class 'Exception'> file: /root/kraken/kraken/plugins/__init__.py line: 247
[chaos:kraken-pod-scenarios] 2024-08-15 14:32:56,644 [INFO]
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,219 [INFO] Telemetry data:
[chaos:kraken-pod-scenarios] {
[chaos:kraken-pod-scenarios]     "scenarios": [
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "startTimeStamp": 1723732014,
[chaos:kraken-pod-scenarios]             "endTimeStamp": 1723732376.6440089,
[chaos:kraken-pod-scenarios]             "scenario": "scenarios/pod_scenario.yaml",
[chaos:kraken-pod-scenarios]             "exitStatus": 1,
[chaos:kraken-pod-scenarios]             "parametersBase64": "",
[chaos:kraken-pod-scenarios]             "parameters": [
[chaos:kraken-pod-scenarios]                 {
[chaos:kraken-pod-scenarios]                     "config": {
[chaos:kraken-pod-scenarios]                         "kill": 1,
[chaos:kraken-pod-scenarios]                         "label_selector": "app=infinispan-router-pod,clusterName=infinispan",
[chaos:kraken-pod-scenarios]                         "namespace_pattern": "runner-keycloak",
[chaos:kraken-pod-scenarios]                         "timeout": 180
[chaos:kraken-pod-scenarios]                     },
[chaos:kraken-pod-scenarios]                     "id": "kill-pods"
[chaos:kraken-pod-scenarios]                 },
[chaos:kraken-pod-scenarios]                 {
[chaos:kraken-pod-scenarios]                     "config": {
[chaos:kraken-pod-scenarios]                         "count": 3,
[chaos:kraken-pod-scenarios]                         "label_selector": "app=infinispan-router-pod,clusterName=infinispan",
[chaos:kraken-pod-scenarios]                         "namespace_pattern": "runner-keycloak",
[chaos:kraken-pod-scenarios]                         "timeout": 360
[chaos:kraken-pod-scenarios]                     },
[chaos:kraken-pod-scenarios]                     "id": "wait-for-pods"
[chaos:kraken-pod-scenarios]                 }
[chaos:kraken-pod-scenarios]             ]
[chaos:kraken-pod-scenarios]         }
[chaos:kraken-pod-scenarios]     ],
[chaos:kraken-pod-scenarios]     "node_infos": [
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "taints": [],
[chaos:kraken-pod-scenarios]             "instance_type": "unknown",
[chaos:kraken-pod-scenarios]             "node_type": "worker",
[chaos:kraken-pod-scenarios]             "name": "ip-10-0-12-12.ec2.internal",
[chaos:kraken-pod-scenarios]             "architecture": "amd64",
[chaos:kraken-pod-scenarios]             "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios]             "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios]             "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios]         },
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "taints": [],
[chaos:kraken-pod-scenarios]             "instance_type": "unknown",
[chaos:kraken-pod-scenarios]             "node_type": "worker",
[chaos:kraken-pod-scenarios]             "name": "ip-10-0-12-13.ec2.internal",
[chaos:kraken-pod-scenarios]             "architecture": "amd64",
[chaos:kraken-pod-scenarios]             "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios]             "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios]             "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios]         },
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "taints": [],
[chaos:kraken-pod-scenarios]             "instance_type": "unknown",
[chaos:kraken-pod-scenarios]             "node_type": "worker",
[chaos:kraken-pod-scenarios]             "name": "ip-10-0-12-4.ec2.internal",
[chaos:kraken-pod-scenarios]             "architecture": "amd64",
[chaos:kraken-pod-scenarios]             "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios]             "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios]             "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios]         },
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "taints": [],
[chaos:kraken-pod-scenarios]             "instance_type": "unknown",
[chaos:kraken-pod-scenarios]             "node_type": "worker",
[chaos:kraken-pod-scenarios]             "name": "ip-10-0-12-5.ec2.internal",
[chaos:kraken-pod-scenarios]             "architecture": "amd64",
[chaos:kraken-pod-scenarios]             "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios]             "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios]             "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios]         },
[chaos:kraken-pod-scenarios]         {
[chaos:kraken-pod-scenarios]             "taints": [],
[chaos:kraken-pod-scenarios]             "instance_type": "unknown",
[chaos:kraken-pod-scenarios]             "node_type": "worker",
[chaos:kraken-pod-scenarios]             "name": "ip-10-0-12-7.ec2.internal",
[chaos:kraken-pod-scenarios]             "architecture": "amd64",
[chaos:kraken-pod-scenarios]             "kernel_version": "5.14.0-284.73.1.el9_2.x86_64",
[chaos:kraken-pod-scenarios]             "kubelet_version": "v1.28.11+add48d0",
[chaos:kraken-pod-scenarios]             "os_version": "Red Hat Enterprise Linux CoreOS 415.92.202407031156-0 (Plow)"
[chaos:kraken-pod-scenarios]         }
[chaos:kraken-pod-scenarios]     ],
[chaos:kraken-pod-scenarios]     "kubernetes_objects_count": {
[chaos:kraken-pod-scenarios]         "ConfigMap": 313,
[chaos:kraken-pod-scenarios]         "Pod": 156,
[chaos:kraken-pod-scenarios]         "Secret": 932,
[chaos:kraken-pod-scenarios]         "Deployment": 38,
[chaos:kraken-pod-scenarios]         "Route": 14,
[chaos:kraken-pod-scenarios]         "Build": 1
[chaos:kraken-pod-scenarios]     },
[chaos:kraken-pod-scenarios]     "network_plugins": [
[chaos:kraken-pod-scenarios]         "OVNKubernetes"
[chaos:kraken-pod-scenarios]     ],
[chaos:kraken-pod-scenarios]     "node_count": 5,
[chaos:kraken-pod-scenarios]     "cloud_infrastructure": "AWS",
[chaos:kraken-pod-scenarios]     "run_uuid": "44039d70-0f8b-4c30-a697-ab93a3c357db"
[chaos:kraken-pod-scenarios] }
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,220 [INFO] telemetry collection disabled, skipping.
[chaos:kraken-pod-scenarios] 2024-08-15 14:33:08,220 [ERROR] Post scenarios are still failing at the end of all iterations
[chaos:kraken-pod-scenarios]  _              _
[chaos:kraken-pod-scenarios] | | ___ __ __ _| | _____ _ __
[chaos:kraken-pod-scenarios] | |/ / '__/ _` | |/ / _ \ '_ \
[chaos:kraken-pod-scenarios] |   <| | | (_| |   <  __/ | | |
[chaos:kraken-pod-scenarios] |_|\_\_|  \__,_|_|\_\___|_| |_|
[chaos:kraken-pod-scenarios]
[chaos:kraken-pod-scenarios]
task: Failed to run task "chaos:kill-gossip-router": exit status 1

Version

keycloak-benchmark latest main

Expected behavior

No response

Actual behavior

No response

How to Reproduce?

Run the below command against a ROSA based Keycloak cluster from the provision/rosa-cross-dc` directory

task chaos:kill-gossip-router

Anything else?

No response