From 7b4f818c64e9649246b4dae12264d4b9513bef28 Mon Sep 17 00:00:00 2001 From: Sarah Bennert Date: Wed, 8 Jan 2025 11:22:02 -0500 Subject: [PATCH] Update uperf process (#840) * uperf: wait for vms update * Update base image * Update pod starting clients check --- Dockerfile | 6 +- image_resources/centos9-appstream.repo | 17 +++++ roles/uperf/tasks/main.yml | 75 ++++++++++++++----- roles/uperf/templates/configmap_script.yml.j2 | 2 +- roles/uperf/templates/workload.yml.j2 | 6 +- roles/uperf/templates/workload_vm.yml.j2 | 4 +- 6 files changed, 82 insertions(+), 28 deletions(-) create mode 100644 image_resources/centos9-appstream.repo diff --git a/Dockerfile b/Dockerfile index 046fda623..5c406d73d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/operator-framework/ansible-operator:v1.32.0 +FROM quay.io/operator-framework/ansible-operator:v1.37.0 USER root COPY requirements.yml ${HOME}/requirements.yml @@ -8,8 +8,8 @@ RUN ansible-galaxy collection list RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ && chmod -R ug+rwx ${HOME}/.ansible -COPY image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.repo -RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream-* && dnf clean all +COPY image_resources/centos9-appstream.repo /etc/yum.repos.d/centos9-appstream.repo +RUN dnf install -y --nodocs redis openssl --enablerepo=centos9-appstream-* --refresh && dnf clean all COPY resources/kernel-cache-drop-daemonset.yaml /opt/kernel_cache_dropper/ COPY watches.yaml ${HOME}/watches.yaml diff --git a/image_resources/centos9-appstream.repo b/image_resources/centos9-appstream.repo new file mode 100644 index 000000000..17bddb897 --- /dev/null +++ b/image_resources/centos9-appstream.repo @@ -0,0 +1,17 @@ +[centos9-appstream-x86_64] +name=CentOS-9-Appstream-x86_64 +baseurl=http://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/ +enabled=0 +gpgcheck=0 + +[centos9-appstream-aarch64] +name=CentOS-9-Appstream-aarch64 +baseurl=http://mirror.stream.centos.org/9-stream/AppStream/aarch64/os/ +enabled=0 +gpgcheck=0 + +[centos9-appstream-ppc64le] +name=CentOS-9-Appstream-ppc64le +baseurl=http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/ +enabled=0 +gpgcheck=0 diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 0c530c6e6..cf2f24952 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -10,12 +10,16 @@ register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip +- name: Get Workload Pairs + set_fact: + workload_pairs: "{{ workload_args.pair|default('1')|int }}" + - block: - name: Create service for server pods k8s: definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + with_sequence: start=0 count={{ workload_pairs }} when: - workload_args.serviceip is defined and workload_args.serviceip - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or @@ -24,7 +28,7 @@ - name: Create metal LB service for server pods k8s: definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + with_sequence: start=0 count={{ workload_pairs }} when: - workload_args.serviceip is defined and workload_args.serviceip - workload_args.servicetype | default("clusterip") == "metallb" @@ -33,7 +37,7 @@ k8s: definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" register: servers - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + with_sequence: start=0 count={{ workload_pairs }} - include_role: name: benchmark_state @@ -49,7 +53,7 @@ k8s: definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}" register: servers - with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + with_sequence: start=0 count={{ workload_pairs }} - include_role: name: benchmark_state @@ -70,19 +74,21 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods + - name: Get number of running server pods + set_fact: + running_pod_count: "{{ server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length|int }}" + - include_role: name: benchmark_state tasks_from: set_state vars: state: "Starting Clients" - when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - + when: workload_pairs == running_pod_count when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" - block: - - - name: Wait for vms to be running.... + - name: Get Server VMs k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -91,19 +97,31 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms + - name: Get Running VMs + set_fact: + running_vm_count: "{{ server_vms | json_query('resources[].status[]') | selectattr('phase','match','Running') | list | length | int }}" + + - name: Get VMs with ip addresses + set_fact: + server_vms_with_ips: "{{ server_vms | json_query('resources[].status.interfaces[0]') | selectattr('ipAddress', 'defined') | map(attribute='ipAddress') | list | length | int }}" + + - name: Determine if can start clients + set_fact: + start_clients: "{{ (workload_pairs == running_vm_count) and (workload_pairs == server_vms_with_ips) }}" + - include_role: name: benchmark_state tasks_from: set_state vars: state: "Starting Clients" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + when: start_clients - name: blocking client from running uperf command: "redis-cli set start-{{ trunc_uuid }} false" with_items: "{{ server_vms.resources }}" - when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" + when: start_clients - when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" - block: @@ -148,11 +166,16 @@ - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 - workload_args.servicetype | default("clusterip") != "nodeport" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" + when: resource_kind == "pod" - block: - - - name: Wait for vms to be running.... + - name: Get Server VMs k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 @@ -161,25 +184,39 @@ - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms + - name: Get Running VMs + set_fact: + running_vm_count: "{{ server_vms | json_query('resources[].status[]') | selectattr('phase','match','Running') | list | length | int }}" + + - name: Get VMs with ip addresses + set_fact: + server_vms_with_ips: "{{ server_vms | json_query('resources[].status.interfaces[0]') | selectattr('ipAddress', 'defined') | map(attribute='ipAddress') | list | length | int }}" + + - name: Determine if can start clients + set_fact: + start_clients: "{{ (workload_pairs == running_vm_count) and (workload_pairs == server_vms_with_ips) }}" - name: Generate uperf test files k8s: definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" with_items: "{{ server_vms.resources }}" + when: start_clients - name: Start Client(s) k8s: definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" with_indexed_items: "{{ server_vms.resources }}" - when: server_vms.resources|length > 0 + when: start_clients + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" + when: start_clients when: resource_kind == "vm" - - include_role: - name: benchmark_state - tasks_from: set_state - vars: - state: "Waiting for Clients" when: benchmark_state.resources[0].status.state == "Starting Clients" diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index 2977a4f55..fb8008f23 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: uperf-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + name: uperf-run-script-{{item.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' data: run_script.sh : | diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index fdcf783ab..b61378ab9 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -7,12 +7,12 @@ metadata: {% if workload_args.servicetype | default("clusterip") == "nodeport" %} name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} - name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip | replace(".", "-") }}-{{ trunc_uuid }}' {% else %} - name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.spec.clusterIP | replace(".", "-")}}-{{ trunc_uuid }}' {% endif %} {% else %} - name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.status.podIP | replace(".", "-")}}-{{ trunc_uuid }}' {% endif %} namespace: '{{ operator_namespace }}' {% if workload_args.annotations is defined or workload_args.server_annotations is defined %} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index 35808f20a..c63d520b4 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -2,7 +2,7 @@ apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstance metadata: - name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: benchmark-uuid: {{ uuid }} @@ -93,6 +93,6 @@ spec: name: uperf-test-{{ item.0 }}-{{ trunc_uuid }} name: app-config-disk - configMap: - name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }} name: run-config-disk status: {} \ No newline at end of file