From 4f5fb041f6fcc2ed2adfef94b872b821adca8c58 Mon Sep 17 00:00:00 2001 From: Judy Ng Date: Tue, 5 Dec 2023 16:59:54 -0500 Subject: [PATCH 1/5] Decouple OSU test from EFA tests Signed-off-by: Judy Ng --- tests/integration-tests/configs/osu.yaml | 4 +- tests/integration-tests/tests/efa/test_efa.py | 235 +-------------- .../tests/performance_tests/test_osu.py | 267 ++++++++++++++++++ .../c5n.18xlarge/intelmpi/osu_allgather | 0 .../c5n.18xlarge/intelmpi/osu_allreduce | 0 .../c5n.18xlarge/intelmpi/osu_alltoall | 0 .../alinux2/c5n.18xlarge/intelmpi/osu_bcast | 0 .../alinux2/c5n.18xlarge/intelmpi/osu_bibw | 0 .../alinux2/c5n.18xlarge/intelmpi/osu_latency | 0 .../c5n.18xlarge/openmpi/osu_allgather | 0 .../c5n.18xlarge/openmpi/osu_allreduce | 0 .../alinux2/c5n.18xlarge/openmpi/osu_alltoall | 0 .../alinux2/c5n.18xlarge/openmpi/osu_bcast | 0 .../alinux2/c5n.18xlarge/openmpi/osu_bibw | 0 .../alinux2/c5n.18xlarge/openmpi/osu_latency | 0 .../p5.48xlarge/intelmpi/osu_allgather | 0 .../p5.48xlarge/intelmpi/osu_allreduce | 0 .../alinux2/p5.48xlarge/intelmpi/osu_alltoall | 0 .../alinux2/p5.48xlarge/intelmpi/osu_bcast | 0 .../alinux2/p5.48xlarge/intelmpi/osu_bibw | 0 .../alinux2/p5.48xlarge/intelmpi/osu_latency | 0 .../alinux2/p5.48xlarge/openmpi/osu_allgather | 0 .../alinux2/p5.48xlarge/openmpi/osu_allreduce | 0 .../alinux2/p5.48xlarge/openmpi/osu_alltoall | 0 .../alinux2/p5.48xlarge/openmpi/osu_bcast | 0 .../alinux2/p5.48xlarge/openmpi/osu_bibw | 0 .../alinux2/p5.48xlarge/openmpi/osu_latency | 0 .../c5n.18xlarge/intelmpi/osu_allgather | 0 .../c5n.18xlarge/intelmpi/osu_allreduce | 0 .../c5n.18xlarge/intelmpi/osu_alltoall | 0 .../centos7/c5n.18xlarge/intelmpi/osu_bcast | 0 .../centos7/c5n.18xlarge/intelmpi/osu_bibw | 0 .../centos7/c5n.18xlarge/intelmpi/osu_latency | 0 .../c5n.18xlarge/openmpi/osu_allgather | 0 .../c5n.18xlarge/openmpi/osu_allreduce | 0 .../centos7/c5n.18xlarge/openmpi/osu_alltoall | 0 .../centos7/c5n.18xlarge/openmpi/osu_bcast | 0 .../centos7/c5n.18xlarge/openmpi/osu_bibw | 0 .../centos7/c5n.18xlarge/openmpi/osu_latency | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_allgather | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_allreduce | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_alltoall | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_bcast | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_bibw | 0 .../rhel8/c5n.18xlarge/intelmpi/osu_latency | 0 .../rhel8/c5n.18xlarge/openmpi/osu_allgather | 0 .../rhel8/c5n.18xlarge/openmpi/osu_allreduce | 0 .../rhel8/c5n.18xlarge/openmpi/osu_alltoall | 0 .../rhel8/c5n.18xlarge/openmpi/osu_bcast | 0 .../rhel8/c5n.18xlarge/openmpi/osu_bibw | 0 .../rhel8/c5n.18xlarge/openmpi/osu_latency | 0 .../c5n.18xlarge/intelmpi/osu_allgather | 0 .../c5n.18xlarge/intelmpi/osu_allreduce | 0 .../rocky8/c5n.18xlarge/intelmpi/osu_alltoall | 0 .../rocky8/c5n.18xlarge/intelmpi/osu_bcast | 0 .../rocky8/c5n.18xlarge/intelmpi/osu_bibw | 0 .../rocky8/c5n.18xlarge/intelmpi/osu_latency | 0 .../rocky8/c5n.18xlarge/openmpi/osu_allgather | 0 .../rocky8/c5n.18xlarge/openmpi/osu_allreduce | 0 .../rocky8/c5n.18xlarge/openmpi/osu_alltoall | 0 .../rocky8/c5n.18xlarge/openmpi/osu_bcast | 0 .../rocky8/c5n.18xlarge/openmpi/osu_bibw | 0 .../rocky8/c5n.18xlarge/openmpi/osu_latency | 0 .../c5n.18xlarge/intelmpi/osu_allgather | 0 .../c5n.18xlarge/intelmpi/osu_allreduce | 0 .../c5n.18xlarge/intelmpi/osu_alltoall | 0 .../c5n.18xlarge/intelmpi/osu_bcast | 0 .../ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw | 0 .../c5n.18xlarge/intelmpi/osu_latency | 0 .../c5n.18xlarge/openmpi/osu_allgather | 0 .../c5n.18xlarge/openmpi/osu_allreduce | 0 .../c5n.18xlarge/openmpi/osu_alltoall | 0 .../ubuntu2004/c5n.18xlarge/openmpi/osu_bcast | 0 .../ubuntu2004/c5n.18xlarge/openmpi/osu_bibw | 0 .../c5n.18xlarge/openmpi/osu_latency | 0 .../c5n.18xlarge/intelmpi/osu_allgather | 0 .../c5n.18xlarge/intelmpi/osu_allreduce | 0 .../c5n.18xlarge/intelmpi/osu_alltoall | 0 .../c5n.18xlarge/intelmpi/osu_bcast | 0 .../ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw | 0 .../c5n.18xlarge/intelmpi/osu_latency | 0 .../c5n.18xlarge/openmpi/osu_allgather | 0 .../c5n.18xlarge/openmpi/osu_allreduce | 0 .../c5n.18xlarge/openmpi/osu_alltoall | 0 .../ubuntu2204/c5n.18xlarge/openmpi/osu_bcast | 0 .../ubuntu2204/c5n.18xlarge/openmpi/osu_bibw | 0 .../c5n.18xlarge/openmpi/osu_latency | 0 .../test_osu/test_osu/pcluster.config.yaml | 37 +++ 88 files changed, 314 insertions(+), 229 deletions(-) create mode 100644 tests/integration-tests/tests/performance_tests/test_osu.py rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_latency (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allgather (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allreduce (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_alltoall (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bcast (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bibw (100%) rename tests/integration-tests/tests/{efa/test_efa/test_efa => performance_tests/test_osu/test_osu}/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_latency (100%) create mode 100644 tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml diff --git a/tests/integration-tests/configs/osu.yaml b/tests/integration-tests/configs/osu.yaml index 6351cb4830..00b65544c2 100644 --- a/tests/integration-tests/configs/osu.yaml +++ b/tests/integration-tests/configs/osu.yaml @@ -1,7 +1,7 @@ {%- import 'common.jinja2' as common with context -%} test-suites: - efa: - test_efa.py::test_efa: + performance_tests: + test_osu.py::test_osu: dimensions: - regions: [ "euw1-az1" ] # do not move, unless capacity reservation is moved as well instances: [ "c5n.18xlarge" ] diff --git a/tests/integration-tests/tests/efa/test_efa.py b/tests/integration-tests/tests/efa/test_efa.py index 4c2b83498a..7f5bb2ec69 100644 --- a/tests/integration-tests/tests/efa/test_efa.py +++ b/tests/integration-tests/tests/efa/test_efa.py @@ -10,9 +10,7 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. import logging -import re -import boto3 import pytest from assertpy import assert_that from remote_command_executor import RemoteCommandExecutor @@ -20,8 +18,7 @@ from tests.common.assertions import assert_no_errors_in_logs from tests.common.mpi_common import _test_mpi -from tests.common.osu_common import run_individual_osu_benchmark -from tests.common.utils import fetch_instance_slots, get_installed_parallelcluster_version, run_system_analyzer +from tests.common.utils import fetch_instance_slots, run_system_analyzer @pytest.mark.usefixtures("serial_execution_by_instance") @@ -34,8 +31,6 @@ def test_efa( clusters_factory, test_datadir, architecture, - network_interfaces_count, - mpi_variants, scheduler_commands_factory, request, ): @@ -44,11 +39,8 @@ def test_efa( Grouped all tests in a single function so that cluster can be reused for all of them. """ - # We collected OSU benchmarks results for c5n.18xlarge only. - osu_benchmarks_instances = ["c5n.18xlarge"] - # 32 instances are required to see performance differences in collective OSU benchmarks. - max_queue_size = 32 if instance in osu_benchmarks_instances else 2 + max_queue_size = 2 if architecture == "x86_64": head_node_instance = "c5.18xlarge" @@ -61,63 +53,16 @@ def test_efa( remote_command_executor = RemoteCommandExecutor(cluster) scheduler_commands = scheduler_commands_factory(remote_command_executor) - # Run EFA tests if not running OSU benchmark, in future decouple EFA from OSU tests - # TODO: Remove this condition once OSU benchmark tests are decoupled from EFA tests - if instance not in osu_benchmarks_instances or os == "rocky8": - _test_efa_installation(scheduler_commands, remote_command_executor, efa_installed=True, partition="efa-enabled") - _test_mpi(remote_command_executor, slots_per_instance, scheduler, scheduler_commands, partition="efa-enabled") - logging.info("Running on Instances: {0}".format(get_compute_nodes_instance_ids(cluster.cfn_name, region))) + _test_efa_installation(scheduler_commands, remote_command_executor, efa_installed=True, partition="efa-enabled") + _test_mpi(remote_command_executor, slots_per_instance, scheduler, scheduler_commands, partition="efa-enabled") + logging.info("Running on Instances: {0}".format(get_compute_nodes_instance_ids(cluster.cfn_name, region))) run_system_analyzer(cluster, scheduler_commands_factory, request, partition="efa-enabled") - if instance in osu_benchmarks_instances: - benchmark_failures = [] + _test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor, partition="efa-enabled") - # Run OSU benchmarks in efa-enabled queue. - for mpi_version in mpi_variants: - benchmark_failures.extend( - _test_osu_benchmarks_pt2pt( - mpi_version, - remote_command_executor, - scheduler_commands, - test_datadir, - os, - instance, - slots_per_instance, - partition="efa-enabled", - ) - ) - benchmark_failures.extend( - _test_osu_benchmarks_collective( - mpi_version, - remote_command_executor, - scheduler_commands, - test_datadir, - os, - instance, - num_instances=max_queue_size, - slots_per_instance=slots_per_instance, - partition="efa-enabled", - ) - ) - assert_that(benchmark_failures, description="Some OSU benchmarks are failing").is_empty() - - if network_interfaces_count > 1: - _test_osu_benchmarks_multiple_bandwidth( - instance, - remote_command_executor, - scheduler_commands, - test_datadir, - slots_per_instance, - partition="efa-enabled", - ) - - # TODO: Remove this condition once OSU benchmark tests are decoupled from EFA tests - if instance not in osu_benchmarks_instances or os == "rocky8": - _test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor, partition="efa-enabled") - - if instance == "p4d.24xlarge" and os != "centos7": - _test_nccl_benchmarks(remote_command_executor, test_datadir, "openmpi", scheduler_commands) + if instance == "p4d.24xlarge" and os != "centos7": + _test_nccl_benchmarks(remote_command_executor, test_datadir, "openmpi", scheduler_commands) assert_no_errors_in_logs(remote_command_executor, scheduler, skip_ice=True) @@ -147,170 +92,6 @@ def _test_efa_installation(scheduler_commands, remote_command_executor, efa_inst assert_that(result.stdout).does_not_contain("1d0f:efa") -def _test_osu_benchmarks_pt2pt( - mpi_version, - remote_command_executor, - scheduler_commands, - test_datadir, - os, - instance, - slots_per_instance, - partition=None, -): - # OSU pt2pt benchmarks cannot be executed with more than 2 MPI ranks. - # Run them in 2 instances with 1 proc per instance, defined by map-by parameter. - num_instances = 2 - # Accept a max number of 4 failures on a total of 23-24 packet size tests. - accepted_number_of_failures = 4 - - failed_benchmarks = [] - for benchmark_name in ["osu_latency", "osu_bibw"]: - _, output = run_individual_osu_benchmark( - mpi_version, - "pt2pt", - benchmark_name, - partition, - remote_command_executor, - scheduler_commands, - num_instances, - slots_per_instance, - test_datadir, - ) - failures = _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output) - if failures > accepted_number_of_failures: - failed_benchmarks.append(f"{mpi_version}-{benchmark_name}") - - return failed_benchmarks - - -def _test_osu_benchmarks_collective( - mpi_version, - remote_command_executor, - scheduler_commands, - test_datadir, - os, - instance, - num_instances, - slots_per_instance, - partition=None, -): - # Accept a max number of 3 failures on a total of 19-21 packet size tests. - accepted_number_of_failures = 3 - - failed_benchmarks = [] - for benchmark_name in ["osu_allgather", "osu_bcast", "osu_allreduce", "osu_alltoall"]: - _, output = run_individual_osu_benchmark( - mpi_version, - "collective", - benchmark_name, - partition, - remote_command_executor, - scheduler_commands, - num_instances, - slots_per_instance, - test_datadir, - timeout=24, - ) - failures = _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output) - if failures > accepted_number_of_failures: - failed_benchmarks.append(f"{mpi_version}-{benchmark_name}") - - return failed_benchmarks - - -def _test_osu_benchmarks_multiple_bandwidth( - instance, remote_command_executor, scheduler_commands, test_datadir, slots_per_instance, partition=None -): - instance_bandwidth_dict = { - # Expected bandwidth for p4d and p4de (4 * 100 Gbps NICS -> declared NetworkPerformance 400 Gbps): - # OMPI 4.1.0: ~330Gbps = 41250MB/s with Placement Group - # OMPI 4.1.0: ~252Gbps = 31550MB/s without Placement Group - # OMPI 4.0.5: ~95Gbps = 11875MB/s with Placement Group - "p4d.24xlarge": 30000, # Equivalent to a theoretical maximum of a single 240Gbps card - # 4 100 Gbps NICS -> declared NetworkPerformance 400 Gbps - "p4de.24xlarge": 30000, # Equivalent to a theoretical maximum of a single 240Gbps card - # 2 up to 170 Gbps NICS -> declared NetworkPerformance 200 Gbps - "hpc6id.32xlarge": 23000, # Equivalent to a theoretical maximum of a single 184Gbps card - # 8 100 Gbps NICS -> declared NetworkPerformance 800 Gbps - "trn1.32xlarge": 80000, # Equivalent to a theoretical maximum of a single 640Gbps card - } - num_instances = 2 - run_individual_osu_benchmark( - "openmpi", - "mbw_mr", - "osu_mbw_mr", - partition, - remote_command_executor, - scheduler_commands, - num_instances, - slots_per_instance, - test_datadir, - ) - max_bandwidth = remote_command_executor.run_remote_command( - "cat /shared/osu_mbw_mr.out | tail -n +4 | awk '{print $2}' | sort -n | tail -n 1" - ).stdout - - expected_bandwidth = instance_bandwidth_dict.get(instance) - if expected_bandwidth is None: - pytest.fail(f"Instance {instance} is not valid for multiple bandwidth tests") - - assert_that(float(max_bandwidth)).is_greater_than(expected_bandwidth) - - -def _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output): - logging.info(output) - # Check avg latency for all packet sizes - failures = 0 - metric_data = [] - metric_namespace = "ParallelCluster/test_efa" - for packet_size, value in re.findall(r"(\d+)\s+(\d+)\.", output): - with open( - str(test_datadir / "osu_benchmarks" / "results" / os / instance / mpi_version / benchmark_name), - encoding="utf-8", - ) as result: - previous_result = re.search(rf"{packet_size}\s+(\d+)\.", result.read()).group(1) - - if benchmark_name == "osu_bibw": - # Invert logic because osu_bibw is in MB/s - tolerated_value = float(previous_result) - (float(previous_result) * 0.2) - is_failure = int(value) < tolerated_value - else: - multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2 - tolerated_value = float(previous_result) + max(float(previous_result) * multiplier, 10) - - is_failure = int(value) > tolerated_value - - message = ( - f"{mpi_version} - {benchmark_name} - packet size {packet_size}: " - f"tolerated: {tolerated_value}, current: {value}" - ) - - dimensions = { - "PclusterVersion": get_installed_parallelcluster_version(), - "MpiVariant": mpi_version, - "Instance": instance, - "OsuBenchmarkName": benchmark_name, - "PacketSize": packet_size, - } - metric_data.append( - { - "MetricName": "Latency", - "Dimensions": [{"Name": name, "Value": str(value)} for name, value in dimensions.items()], - "Value": int(value), - "Unit": "Microseconds", - } - ) - - if is_failure: - failures = failures + 1 - logging.error(message) - else: - logging.info(message) - boto3.client("cloudwatch").put_metric_data(Namespace=metric_namespace, MetricData=metric_data) - - return failures - - def _test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor, partition=None): logging.info("Testing SHM Transfer is enabled") if partition: diff --git a/tests/integration-tests/tests/performance_tests/test_osu.py b/tests/integration-tests/tests/performance_tests/test_osu.py new file mode 100644 index 0000000000..205a063baf --- /dev/null +++ b/tests/integration-tests/tests/performance_tests/test_osu.py @@ -0,0 +1,267 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. +import logging +import re + +import boto3 +import pytest +from assertpy import assert_that +from remote_command_executor import RemoteCommandExecutor + +from tests.common.assertions import assert_no_errors_in_logs +from tests.common.osu_common import run_individual_osu_benchmark +from tests.common.utils import fetch_instance_slots, get_installed_parallelcluster_version, run_system_analyzer + + +@pytest.mark.usefixtures("serial_execution_by_instance") +def test_osu( + os, + region, + scheduler, + instance, + pcluster_config_reader, + clusters_factory, + test_datadir, + architecture, + network_interfaces_count, + mpi_variants, + scheduler_commands_factory, + request, +): + # We collected OSU benchmarks results for c5n.18xlarge only. + osu_benchmarks_instances = ["c5n.18xlarge"] + + if instance not in osu_benchmarks_instances: + raise Exception(f"OSU benchmarks can't be run on instance {instance}.") + + # 32 instances are required to see performance differences in collective OSU benchmarks. + max_queue_size = 32 + + if architecture == "x86_64": + head_node_instance = "c5.18xlarge" + else: + head_node_instance = "c6g.16xlarge" + + slots_per_instance = fetch_instance_slots(region, instance, multithreading_disabled=True) + cluster_config = pcluster_config_reader(max_queue_size=max_queue_size, head_node_instance=head_node_instance) + cluster = clusters_factory(cluster_config) + remote_command_executor = RemoteCommandExecutor(cluster) + scheduler_commands = scheduler_commands_factory(remote_command_executor) + + run_system_analyzer(cluster, scheduler_commands_factory, request, partition="efa-enabled") + + benchmark_failures = [] + + # Run OSU benchmarks in efa-enabled queue. + for mpi_version in mpi_variants: + benchmark_failures.extend( + _test_osu_benchmarks_pt2pt( + mpi_version, + remote_command_executor, + scheduler_commands, + test_datadir, + os, + instance, + slots_per_instance, + partition="efa-enabled", + ) + ) + benchmark_failures.extend( + _test_osu_benchmarks_collective( + mpi_version, + remote_command_executor, + scheduler_commands, + test_datadir, + os, + instance, + num_instances=max_queue_size, + slots_per_instance=slots_per_instance, + partition="efa-enabled", + ) + ) + assert_that(benchmark_failures, description="Some OSU benchmarks are failing").is_empty() + + if network_interfaces_count > 1: + _test_osu_benchmarks_multiple_bandwidth( + instance, + remote_command_executor, + scheduler_commands, + test_datadir, + slots_per_instance, + partition="efa-enabled", + ) + + assert_no_errors_in_logs(remote_command_executor, scheduler, skip_ice=True) + + +def _test_osu_benchmarks_pt2pt( + mpi_version, + remote_command_executor, + scheduler_commands, + test_datadir, + os, + instance, + slots_per_instance, + partition=None, +): + # OSU pt2pt benchmarks cannot be executed with more than 2 MPI ranks. + # Run them in 2 instances with 1 proc per instance, defined by map-by parameter. + num_instances = 2 + # Accept a max number of 4 failures on a total of 23-24 packet size tests. + accepted_number_of_failures = 4 + + failed_benchmarks = [] + for benchmark_name in ["osu_latency", "osu_bibw"]: + _, output = run_individual_osu_benchmark( + mpi_version, + "pt2pt", + benchmark_name, + partition, + remote_command_executor, + scheduler_commands, + num_instances, + slots_per_instance, + test_datadir, + ) + failures = _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output) + if failures > accepted_number_of_failures: + failed_benchmarks.append(f"{mpi_version}-{benchmark_name}") + + return failed_benchmarks + + +def _test_osu_benchmarks_collective( + mpi_version, + remote_command_executor, + scheduler_commands, + test_datadir, + os, + instance, + num_instances, + slots_per_instance, + partition=None, +): + # Accept a max number of 3 failures on a total of 19-21 packet size tests. + accepted_number_of_failures = 3 + + failed_benchmarks = [] + for benchmark_name in ["osu_allgather", "osu_bcast", "osu_allreduce", "osu_alltoall"]: + _, output = run_individual_osu_benchmark( + mpi_version, + "collective", + benchmark_name, + partition, + remote_command_executor, + scheduler_commands, + num_instances, + slots_per_instance, + test_datadir, + timeout=24, + ) + failures = _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output) + if failures > accepted_number_of_failures: + failed_benchmarks.append(f"{mpi_version}-{benchmark_name}") + + return failed_benchmarks + + +def _test_osu_benchmarks_multiple_bandwidth( + instance, remote_command_executor, scheduler_commands, test_datadir, slots_per_instance, partition=None +): + instance_bandwidth_dict = { + # Expected bandwidth for p4d and p4de (4 * 100 Gbps NICS -> declared NetworkPerformance 400 Gbps): + # OMPI 4.1.0: ~330Gbps = 41250MB/s with Placement Group + # OMPI 4.1.0: ~252Gbps = 31550MB/s without Placement Group + # OMPI 4.0.5: ~95Gbps = 11875MB/s with Placement Group + "p4d.24xlarge": 30000, # Equivalent to a theoretical maximum of a single 240Gbps card + # 4 100 Gbps NICS -> declared NetworkPerformance 400 Gbps + "p4de.24xlarge": 30000, # Equivalent to a theoretical maximum of a single 240Gbps card + # 2 up to 170 Gbps NICS -> declared NetworkPerformance 200 Gbps + "hpc6id.32xlarge": 23000, # Equivalent to a theoretical maximum of a single 184Gbps card + # 8 100 Gbps NICS -> declared NetworkPerformance 800 Gbps + "trn1.32xlarge": 80000, # Equivalent to a theoretical maximum of a single 640Gbps card + } + num_instances = 2 + run_individual_osu_benchmark( + "openmpi", + "mbw_mr", + "osu_mbw_mr", + partition, + remote_command_executor, + scheduler_commands, + num_instances, + slots_per_instance, + test_datadir, + ) + max_bandwidth = remote_command_executor.run_remote_command( + "cat /shared/osu_mbw_mr.out | tail -n +4 | awk '{print $2}' | sort -n | tail -n 1" + ).stdout + + expected_bandwidth = instance_bandwidth_dict.get(instance) + if expected_bandwidth is None: + pytest.fail(f"Instance {instance} is not valid for multiple bandwidth tests") + + assert_that(float(max_bandwidth)).is_greater_than(expected_bandwidth) + + +def _check_osu_benchmarks_results(test_datadir, os, instance, mpi_version, benchmark_name, output): + logging.info(output) + # Check avg latency for all packet sizes + failures = 0 + metric_data = [] + metric_namespace = "ParallelCluster/test_efa" + for packet_size, value in re.findall(r"(\d+)\s+(\d+)\.", output): + with open( + str(test_datadir / "osu_benchmarks" / "results" / os / instance / mpi_version / benchmark_name), + encoding="utf-8", + ) as result: + previous_result = re.search(rf"{packet_size}\s+(\d+)\.", result.read()).group(1) + + if benchmark_name == "osu_bibw": + # Invert logic because osu_bibw is in MB/s + tolerated_value = float(previous_result) - (float(previous_result) * 0.2) + is_failure = int(value) < tolerated_value + else: + multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2 + tolerated_value = float(previous_result) + max(float(previous_result) * multiplier, 10) + + is_failure = int(value) > tolerated_value + + message = ( + f"{mpi_version} - {benchmark_name} - packet size {packet_size}: " + f"tolerated: {tolerated_value}, current: {value}" + ) + + dimensions = { + "PclusterVersion": get_installed_parallelcluster_version(), + "MpiVariant": mpi_version, + "Instance": instance, + "OsuBenchmarkName": benchmark_name, + "PacketSize": packet_size, + } + metric_data.append( + { + "MetricName": "Latency", + "Dimensions": [{"Name": name, "Value": str(value)} for name, value in dimensions.items()], + "Value": int(value), + "Unit": "Microseconds", + } + ) + + if is_failure: + failures = failures + 1 + logging.error(message) + else: + logging.info(message) + boto3.client("cloudwatch").put_metric_data(Namespace=metric_namespace, MetricData=metric_data) + + return failures diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/alinux2/p5.48xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/centos7/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rhel8/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/rocky8/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2004/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/intelmpi/osu_latency diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allgather b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allgather similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allgather rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allgather diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allreduce b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allreduce similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allreduce rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_allreduce diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_alltoall b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_alltoall similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_alltoall rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_alltoall diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bcast b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bcast similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bcast rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bcast diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bibw b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bibw similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bibw rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_bibw diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_latency b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_latency similarity index 100% rename from tests/integration-tests/tests/efa/test_efa/test_efa/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_latency rename to tests/integration-tests/tests/performance_tests/test_osu/test_osu/osu_benchmarks/results/ubuntu2204/c5n.18xlarge/openmpi/osu_latency diff --git a/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml new file mode 100644 index 0000000000..36562e4149 --- /dev/null +++ b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml @@ -0,0 +1,37 @@ +Image: + Os: {{ os }} +HeadNode: + InstanceType: {{ head_node_instance }} + Networking: + SubnetId: {{ public_subnet_id }} + Ssh: + KeyName: {{ key_name }} + Imds: + Secured: {{ imds_secured }} +Scheduling: + Scheduler: {{ scheduler }} + SlurmQueues: + - Name: efa-enabled + Networking: + PlacementGroup: + Enabled: true + Name: c5n_capacity_reservation + SubnetIds: + - {{ private_subnet_id }} + ComputeResources: + - Name: efa-enabled-i1 + {% if "us-iso" in region %} + InstanceType: {{ instance }} + {% else %} + Instances: + - InstanceType: {{ instance }} + {% endif %} + MaxCount: {{ max_queue_size }} + MinCount: {{ max_queue_size }} + DisableSimultaneousMultithreading: true + Efa: + Enabled: true +SharedStorage: + - MountDir: /shared + Name: name1 + StorageType: Ebs From d8de6541aa2296d1b0e07102b8fbdcc7819868a6 Mon Sep 17 00:00:00 2001 From: Judy Ng Date: Wed, 6 Dec 2023 14:55:01 -0500 Subject: [PATCH 2/5] Remove running benchmarks from other tests Signed-off-by: Judy Ng --- .../ad_integration/test_ad_integration.py | 25 ++++++------------- .../test_ad_integration/workload.sh | 11 -------- .../test_disable_hyperthreading.py | 5 +--- .../tests/storage/test_efs.py | 17 +++---------- .../tests/storage/test_fsx_lustre.py | 12 +++------ 5 files changed, 14 insertions(+), 56 deletions(-) diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration.py b/tests/integration-tests/tests/ad_integration/test_ad_integration.py index 6ea296c079..68d6656bae 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration.py +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration.py @@ -39,7 +39,6 @@ ) from tests.ad_integration.cluster_user import ClusterUser -from tests.common.osu_common import compile_osu from tests.common.utils import get_sts_endpoint, retrieve_latest_ami, run_system_analyzer from tests.storage.test_fsx_lustre import create_fsx_ontap, create_fsx_open_zfs @@ -427,8 +426,7 @@ def _directory_factory( _delete_certificate(certificate_arn=certificate_arn, region=region) -def _run_user_workloads(users, test_datadir, remote_command_executor, shared_storage_mount_dirs): - compile_osu("openmpi", remote_command_executor) +def _run_user_workloads(users, test_datadir, shared_storage_mount_dirs): _check_whoami(users) _check_files_permissions(users, shared_storage_mount_dirs) job_submission_outputs = [ @@ -659,8 +657,6 @@ def test_ad_integration( request, store_secret_in_secret_manager, clusters_factory, - run_benchmarks, - benchmarks, ): """ Verify AD integration works as expected. @@ -670,13 +666,11 @@ def test_ad_integration( 3. SSH key for AD users is created when the property GenerateSshKeysForUsers is true; 4. AD users can submit workloads; 5. AD users filter out by LdapAccessFilter cannot access to the head node. - - Optionally, it executes performance tests using OSU benchmarks. """ if not is_directory_supported(region, directory_type): pytest.skip(f"Skipping the test because directory type {directory_type} is not supported in region {region}") - head_node_instance_type = "c5n.18xlarge" if request.config.getoption("benchmarks") else "c5.xlarge" + head_node_instance_type = "c5.xlarge" compute_instance_type_info = {"name": "c5.xlarge", "num_cores": 4} fsx_supported = is_fsx_supported(region) config_params = { @@ -711,7 +705,7 @@ def test_ad_integration( ) if fsx_supported: config_params.update(get_fsx_config_param_vals(fsx_factory, svm_factory)) - cluster_config = pcluster_config_reader(benchmarks=benchmarks, **config_params) + cluster_config = pcluster_config_reader(**config_params) cluster = clusters_factory(cluster_config) certificate_secret_arn = nlb_stack_parameters.get("CertificateSecretArn") @@ -758,14 +752,12 @@ def test_ad_integration( shared_storage_mount_dirs = ["/shared", "/efs"] if fsx_supported: shared_storage_mount_dirs.extend(["/fsxlustre", "/fsxontap", "/fsxopenzfs"]) - _run_user_workloads(users, test_datadir, remote_command_executor, shared_storage_mount_dirs) + _run_user_workloads(users, test_datadir, shared_storage_mount_dirs) logging.info("Testing pcluster update and generate ssh keys for user") _check_ssh_key_generation(users[0], remote_command_executor, scheduler_commands, False) # Verify access control with ldap access provider. - updated_config_file = pcluster_config_reader( - config_file="pcluster.config.update.yaml", benchmarks=benchmarks, **config_params - ) + updated_config_file = pcluster_config_reader(config_file="pcluster.config.update.yaml", **config_params) cluster.update(str(updated_config_file), force_update="true") # Reset stateful connection variables after the cluster update remote_command_executor = RemoteCommandExecutor(cluster) @@ -779,9 +771,7 @@ def test_ad_integration( # Verify access control with simple access provider. # With this test we also verify that AdditionalSssdConfigs is working properly. - updated_config_file = pcluster_config_reader( - config_file="pcluster.config.update2.yaml", benchmarks=benchmarks, **config_params - ) + updated_config_file = pcluster_config_reader(config_file="pcluster.config.update2.yaml", **config_params) cluster.update(str(updated_config_file), force_update="true") # Reset stateful connection variables after the cluster update remote_command_executor = RemoteCommandExecutor(cluster) @@ -794,7 +784,6 @@ def test_ad_integration( _check_ssh_auth(user=user, expect_success=user.alias != "PclusterUser0") run_system_analyzer(cluster, scheduler_commands_factory, request) - run_benchmarks(users[0].remote_command_executor(), users[0].scheduler_commands(), diretory_type=directory_type) def _check_ssh_auth(user, expect_success=True): @@ -837,7 +826,7 @@ def test_ad_integration_on_login_nodes( 2. SSH key for AD users is created when the property GenerateSshKeysForUsers is true; 3. AD users can submit workloads; """ - head_node_instance_type = "c5n.18xlarge" if request.config.getoption("benchmarks") else "c5.xlarge" + head_node_instance_type = "c5.xlarge" compute_instance_type_info = {"name": "c5.xlarge", "num_cores": 4} config_params = { "compute_instance_type": compute_instance_type_info.get("name"), diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/workload.sh b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/workload.sh index de715d1fe4..85ff243dcd 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/workload.sh +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/workload.sh @@ -6,14 +6,3 @@ for fspath in shared efs; do # srun has to be used for whoami because slurm_nss plugin only send user information through srun date '+%Y%m%d%H%M%S' > "/$fspath/$(srun whoami)" done - -BENCHMARK_NAME=osu_barrier -OSU_BENCHMARK_VERSION=5.7.1 - -module load openmpi -# Run collective benchmark. The collective operations are close to what a real application looks like. -# NOTE: The test is sized for 4 compute nodes. -# -np total number of processes to run (all CPUs * 4 nodes) -mpirun \ - > /shared/"$(date '+%Y%m%d%H%M%S')-$(srun whoami)-${BENCHMARK_NAME}".out \ - /shared/openmpi/osu-micro-benchmarks-${OSU_BENCHMARK_VERSION}/mpi/collective/${BENCHMARK_NAME} diff --git a/tests/integration-tests/tests/disable_hyperthreading/test_disable_hyperthreading.py b/tests/integration-tests/tests/disable_hyperthreading/test_disable_hyperthreading.py index 04670e1462..ebd75ae0ab 100644 --- a/tests/integration-tests/tests/disable_hyperthreading/test_disable_hyperthreading.py +++ b/tests/integration-tests/tests/disable_hyperthreading/test_disable_hyperthreading.py @@ -28,14 +28,12 @@ def test_hit_disable_hyperthreading( pcluster_config_reader, clusters_factory, default_threads_per_core, - run_benchmarks, - benchmarks, scheduler_commands_factory, request, ): """Test Disable Hyperthreading for HIT clusters.""" slots_per_instance = fetch_instance_slots(region, instance) - cluster_config = pcluster_config_reader(benchmarks=benchmarks) + cluster_config = pcluster_config_reader() cluster = clusters_factory(cluster_config) remote_command_executor = RemoteCommandExecutor(cluster) scheduler_commands = scheduler_commands_factory(remote_command_executor) @@ -60,7 +58,6 @@ def test_hit_disable_hyperthreading( assert_no_errors_in_logs(remote_command_executor, scheduler) run_system_analyzer(cluster, scheduler_commands_factory, request) - run_benchmarks(remote_command_executor, scheduler_commands) def _test_disable_hyperthreading_settings( diff --git a/tests/integration-tests/tests/storage/test_efs.py b/tests/integration-tests/tests/storage/test_efs.py index 2d6dfecbc7..bd55a74761 100644 --- a/tests/integration-tests/tests/storage/test_efs.py +++ b/tests/integration-tests/tests/storage/test_efs.py @@ -114,7 +114,6 @@ def test_multiple_efs( key_name, cfn_stacks_factory, scheduler_commands_factory, - run_benchmarks, ): """ Test when efs_fs_id is provided in the config file, the existing efs can be correctly mounted. @@ -126,11 +125,7 @@ def test_multiple_efs( existing_efs_mount_dirs = [] iam_authorizations = [False, False, True] if scheduler != "awsbatch" else 3 * [False] encryption_in_transits = [False, True, True] if scheduler != "awsbatch" else 3 * [False] - if request.config.getoption("benchmarks") and os == "alinux2": - # Only create more EFS when benchmarks are specified. Limiting OS to reduce cost of too many file systems - num_existing_efs = 20 - else: - num_existing_efs = 3 + num_existing_efs = 3 # create an additional EFS with file system policy to prevent anonymous access existing_efs_ids = efs_stack_factory(num_existing_efs) if scheduler != "awsbatch": @@ -168,14 +163,11 @@ def test_multiple_efs( for i in range(num_existing_efs): existing_efs_mount_dirs.append(f"/existing_efs_mount_dir_{i}") - new_efs_mount_dirs = ["/shared"] # OSU benchmark relies on /shared directory - _assert_subnet_az_relations(region, vpc_stack, expected_in_same_az=False) # change cluster configuration file to test different tls and iam settings to EFS. cluster_config = pcluster_config_reader( existing_efs_mount_dirs=existing_efs_mount_dirs, existing_efs_ids=existing_efs_ids, - new_efs_mount_dirs=new_efs_mount_dirs, iam_authorizations=iam_authorizations, encryption_in_transits=encryption_in_transits, ) @@ -187,17 +179,16 @@ def test_multiple_efs( for i in range(num_existing_efs): remote_command_executor.run_remote_command(f"cat {existing_efs_mount_dirs[i]}/{existing_efs_filenames[i]}") - all_mount_dirs = existing_efs_mount_dirs + new_efs_mount_dirs # append false for the one new_efs_mount_dir iam_authorizations.append(False) encryption_in_transits.append(False) _check_efs_correctly_mounted_and_shared( - all_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits + existing_efs_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits ) if scheduler == "slurm": # Only Slurm supports compute nodes reboot remote_command_executor, scheduler_commands = _check_efs_after_nodes_reboot( - all_mount_dirs, + existing_efs_mount_dirs, cluster, remote_command_executor, scheduler_commands_factory, @@ -205,8 +196,6 @@ def test_multiple_efs( encryption_in_transits, ) - run_benchmarks(remote_command_executor, scheduler_commands) - def _check_efs_after_nodes_reboot( all_mount_dirs, diff --git a/tests/integration-tests/tests/storage/test_fsx_lustre.py b/tests/integration-tests/tests/storage/test_fsx_lustre.py index 7ef17d69ac..3c2f12bc45 100644 --- a/tests/integration-tests/tests/storage/test_fsx_lustre.py +++ b/tests/integration-tests/tests/storage/test_fsx_lustre.py @@ -304,7 +304,6 @@ def test_multiple_fsx( scheduler_commands_factory, test_datadir, request, - run_benchmarks, ): """ Test existing Fsx file system @@ -325,14 +324,10 @@ def test_multiple_fsx( num_existing_fsx_open_zfs_volumes = ( 2 if partition in ["aws"] else 0 ) # China, GovCloud and Isolated do not have OpenZFS. - if request.config.getoption("benchmarks") and os == "alinux2": - # Only create more FSx when benchmarks are specified. Limiting OS to reduce cost of too many file systems - num_existing_fsx = 20 - else: - # Minimal total existing FSx is the number of Ontap and OpenZFS plus one existing FSx Lustre - num_existing_fsx = num_existing_fsx_ontap_volumes + num_existing_fsx_open_zfs_volumes + 1 + # Minimal total existing FSx is the number of Ontap and OpenZFS plus one existing FSx Lustre + num_existing_fsx = num_existing_fsx_ontap_volumes + num_existing_fsx_open_zfs_volumes + 1 num_existing_fsx_lustre = num_existing_fsx - num_existing_fsx_ontap_volumes - num_existing_fsx_open_zfs_volumes - fsx_lustre_mount_dirs = ["/shared"] # OSU benchmark relies on /shared directory + fsx_lustre_mount_dirs = [] for i in range(num_new_fsx_lustre + num_existing_fsx_lustre - 1): fsx_lustre_mount_dirs.append(f"/fsx_lustre_mount_dir{i}") @@ -368,7 +363,6 @@ def test_multiple_fsx( remote_command_executor = RemoteCommandExecutor(cluster) scheduler_commands = scheduler_commands_factory(remote_command_executor) - run_benchmarks(remote_command_executor, scheduler_commands) @pytest.mark.usefixtures("instance") From 4695ec4cb5613caf5187466122f8d986c261a115 Mon Sep 17 00:00:00 2001 From: Judy Ng Date: Thu, 7 Dec 2023 18:17:52 -0500 Subject: [PATCH 3/5] Remove the benchmark config sections Signed-off-by: Judy Ng --- .../configs/ad_integration.yaml | 7 ------ .../configs/common/common.yaml | 24 ------------------- .../configs/isolated_regions.yaml | 18 -------------- tests/integration-tests/configs/new_os.yaml | 24 ------------------- 4 files changed, 73 deletions(-) diff --git a/tests/integration-tests/configs/ad_integration.yaml b/tests/integration-tests/configs/ad_integration.yaml index bc9fe5c269..c75bc49077 100644 --- a/tests/integration-tests/configs/ad_integration.yaml +++ b/tests/integration-tests/configs/ad_integration.yaml @@ -8,10 +8,3 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["alinux2", "ubuntu2004"] schedulers: ["slurm"] - benchmarks: - - mpi_variants: ["openmpi"] - num_instances: [100] - osu_benchmarks: - # Available collective benchmarks "osu_allgather", "osu_allreduce", "osu_alltoall", "osu_barrier", "osu_bcast", "osu_gather", "osu_reduce", "osu_reduce_scatter", "osu_scatter" - collective: ["osu_allreduce", "osu_alltoall"] - pt2pt: [] diff --git a/tests/integration-tests/configs/common/common.yaml b/tests/integration-tests/configs/common/common.yaml index 6910ca0232..f99d54b496 100644 --- a/tests/integration-tests/configs/common/common.yaml +++ b/tests/integration-tests/configs/common/common.yaml @@ -5,11 +5,6 @@ ad_integration: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["alinux2", "rhel8"] schedulers: ["slurm"] - benchmarks: - - mpi_variants: ["openmpi"] - num_instances: [5] - osu_benchmarks: - collective: ["osu_alltoall"] - regions: ["eu-west-1"] instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["ubuntu2204", "centos7"] @@ -240,13 +235,6 @@ disable_hyperthreading: instances: ["c5.xlarge"] oss: ["ubuntu2204"] schedulers: ["slurm"] - benchmarks: - - mpi_variants: ["openmpi", "intelmpi"] - num_instances: [5] # Change the head node instance type if you'd test more than 30 instances - slots_per_instance: 2 - partition: "ht-disabled" - osu_benchmarks: - collective: ["osu_allreduce", "osu_alltoall"] dns: test_dns.py::test_hit_no_cluster_dns_mpi: dimensions: @@ -569,12 +557,6 @@ storage: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["rhel8"] schedulers: ["slurm"] - benchmarks: - - mpi_variants: ["openmpi"] - num_instances: [5] # Change the head node instance type if you'd test more than 30 instances - slots_per_instance: 2 - osu_benchmarks: - collective: ["osu_alltoall"] - regions: ["eu-west-2"] instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: ["ubuntu2004"] @@ -630,12 +612,6 @@ storage: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: ["ubuntu2204"] schedulers: [ "slurm" ] - benchmarks: - - mpi_variants: ["intelmpi"] - num_instances: [5] # Change the head node instance type if you'd test more than 30 instances - slots_per_instance: 2 - osu_benchmarks: - collective: ["osu_alltoall"] - regions: ["us-gov-east-1"] instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["rhel8"] diff --git a/tests/integration-tests/configs/isolated_regions.yaml b/tests/integration-tests/configs/isolated_regions.yaml index 72a3ebf3a1..1dee2dab56 100644 --- a/tests/integration-tests/configs/isolated_regions.yaml +++ b/tests/integration-tests/configs/isolated_regions.yaml @@ -16,11 +16,6 @@ test-suites: instances: {{ INSTANCES }} oss: {{ OSS }} schedulers: {{ SCHEDULERS }} - benchmarks: - - mpi_variants: ["openmpi"] - num_instances: [5] - osu_benchmarks: - collective: ["osu_alltoall"] # This test cannot be executed in US isolated regions because it relies on a CloudFormation stack using resources # that are not supported by CloudFormation in ADC, i.e. CapacityReservation and ResourceGroup. # capacity_reservations: @@ -153,13 +148,6 @@ test-suites: instances: {{ INSTANCES }} oss: {{ OSS }} schedulers: {{ SCHEDULERS }} - benchmarks: - - mpi_variants: ["openmpi", "intelmpi"] - num_instances: [20] # Change the head node instance type if you'd test more than 30 instances - slots_per_instance: 2 - partition: "ht-disabled" - osu_benchmarks: - collective: ["osu_allreduce", "osu_alltoall"] dns: test_dns.py::test_hit_no_cluster_dns_mpi: dimensions: @@ -460,12 +448,6 @@ test-suites: instances: {{ INSTANCES }} oss: {{ OSS }} schedulers: {{ SCHEDULERS }} - benchmarks: - - mpi_variants: ["openmpi", "intelmpi"] - num_instances: [20] # Change the head node instance type if you'd test more than 30 instances - slots_per_instance: 2 - osu_benchmarks: - collective: ["osu_allreduce", "osu_alltoall"] test_raid.py::test_raid_fault_tolerance_mode: dimensions: - regions: {{ REGIONS }} diff --git a/tests/integration-tests/configs/new_os.yaml b/tests/integration-tests/configs/new_os.yaml index 9bf295efbf..fd5968770f 100644 --- a/tests/integration-tests/configs/new_os.yaml +++ b/tests/integration-tests/configs/new_os.yaml @@ -16,12 +16,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: {{ NEW_OS }} schedulers: ["slurm"] - benchmarks: - - mpi_variants: [ "openmpi", "intelmpi" ] - num_instances: [ 4 ] - slots_per_instance: 2 - osu_benchmarks: - collective: [ "osu_alltoall" ] arm_pl: test_arm_pl.py::test_arm_pl: dimensions: @@ -100,12 +94,6 @@ test-suites: instances: ["m4.xlarge"] oss: {{ NEW_OS }} schedulers: ["slurm"] - benchmarks: - - mpi_variants: [ "openmpi", "intelmpi" ] - num_instances: [ 4 ] - slots_per_instance: 2 - osu_benchmarks: - collective: [ "osu_alltoall" ] dns: test_dns.py::test_hit_no_cluster_dns_mpi: dimensions: @@ -237,12 +225,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: {{ NEW_OS }} schedulers: ["slurm"] - benchmarks: - - mpi_variants: [ "openmpi", "intelmpi" ] - num_instances: [ 4 ] - slots_per_instance: 2 - osu_benchmarks: - collective: [ "osu_alltoall" ] test_fsx_lustre.py::test_fsx_lustre_configuration_options: dimensions: - regions: ["us-east-2"] @@ -261,12 +243,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: {{ NEW_OS }} schedulers: [ "slurm" ] - benchmarks: - - mpi_variants: [ "openmpi", "intelmpi" ] - num_instances: [ 4 ] - slots_per_instance: 2 - osu_benchmarks: - collective: [ "osu_alltoall" ] test_raid.py::test_raid_performance_mode: dimensions: - regions: ["ap-south-1"] From e90e55ddda0f08271ec70b4978ed9eebfcc3556e Mon Sep 17 00:00:00 2001 From: Judy Ng Date: Fri, 8 Dec 2023 17:37:12 -0500 Subject: [PATCH 4/5] Add back shared mount dir since used in cluster config Signed-off-by: Judy Ng --- tests/integration-tests/tests/storage/test_efs.py | 8 ++++++-- tests/integration-tests/tests/storage/test_fsx_lustre.py | 5 +---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration-tests/tests/storage/test_efs.py b/tests/integration-tests/tests/storage/test_efs.py index bd55a74761..8d93445276 100644 --- a/tests/integration-tests/tests/storage/test_efs.py +++ b/tests/integration-tests/tests/storage/test_efs.py @@ -163,11 +163,14 @@ def test_multiple_efs( for i in range(num_existing_efs): existing_efs_mount_dirs.append(f"/existing_efs_mount_dir_{i}") + new_efs_mount_dirs = ["/shared"] + _assert_subnet_az_relations(region, vpc_stack, expected_in_same_az=False) # change cluster configuration file to test different tls and iam settings to EFS. cluster_config = pcluster_config_reader( existing_efs_mount_dirs=existing_efs_mount_dirs, existing_efs_ids=existing_efs_ids, + new_efs_mount_dirs=new_efs_mount_dirs, iam_authorizations=iam_authorizations, encryption_in_transits=encryption_in_transits, ) @@ -179,16 +182,17 @@ def test_multiple_efs( for i in range(num_existing_efs): remote_command_executor.run_remote_command(f"cat {existing_efs_mount_dirs[i]}/{existing_efs_filenames[i]}") + all_mount_dirs = existing_efs_mount_dirs + new_efs_mount_dirs # append false for the one new_efs_mount_dir iam_authorizations.append(False) encryption_in_transits.append(False) _check_efs_correctly_mounted_and_shared( - existing_efs_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits + all_mount_dirs, remote_command_executor, scheduler_commands, iam_authorizations, encryption_in_transits ) if scheduler == "slurm": # Only Slurm supports compute nodes reboot remote_command_executor, scheduler_commands = _check_efs_after_nodes_reboot( - existing_efs_mount_dirs, + all_mount_dirs, cluster, remote_command_executor, scheduler_commands_factory, diff --git a/tests/integration-tests/tests/storage/test_fsx_lustre.py b/tests/integration-tests/tests/storage/test_fsx_lustre.py index 3c2f12bc45..dfeacdc81e 100644 --- a/tests/integration-tests/tests/storage/test_fsx_lustre.py +++ b/tests/integration-tests/tests/storage/test_fsx_lustre.py @@ -327,7 +327,7 @@ def test_multiple_fsx( # Minimal total existing FSx is the number of Ontap and OpenZFS plus one existing FSx Lustre num_existing_fsx = num_existing_fsx_ontap_volumes + num_existing_fsx_open_zfs_volumes + 1 num_existing_fsx_lustre = num_existing_fsx - num_existing_fsx_ontap_volumes - num_existing_fsx_open_zfs_volumes - fsx_lustre_mount_dirs = [] + fsx_lustre_mount_dirs = ["/shared"] for i in range(num_new_fsx_lustre + num_existing_fsx_lustre - 1): fsx_lustre_mount_dirs.append(f"/fsx_lustre_mount_dir{i}") @@ -361,9 +361,6 @@ def test_multiple_fsx( bucket_name, ) - remote_command_executor = RemoteCommandExecutor(cluster) - scheduler_commands = scheduler_commands_factory(remote_command_executor) - @pytest.mark.usefixtures("instance") def test_file_cache( From 823f8953728af9caf1ea121552bab57c05ddf478 Mon Sep 17 00:00:00 2001 From: Judy Ng Date: Tue, 12 Dec 2023 14:28:57 -0500 Subject: [PATCH 5/5] Clean up osu decoupling tests Signed-off-by: Judy Ng --- .../ad_integration/test_ad_integration.py | 28 ++++++------------- .../pcluster.config.update.yaml | 4 +-- .../pcluster.config.update2.yaml | 4 +-- .../test_ad_integration/pcluster.config.yaml | 4 +-- .../pcluster.config.yaml | 4 +-- tests/integration-tests/tests/efa/test_efa.py | 5 +--- .../test_efa/test_efa/pcluster.config.yaml | 4 +-- .../tests/performance_tests/test_osu.py | 22 +++++++-------- .../test_osu/test_osu/pcluster.config.yaml | 4 +-- 9 files changed, 32 insertions(+), 47 deletions(-) diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration.py b/tests/integration-tests/tests/ad_integration/test_ad_integration.py index 68d6656bae..e5ce3994af 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration.py +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration.py @@ -670,12 +670,8 @@ def test_ad_integration( if not is_directory_supported(region, directory_type): pytest.skip(f"Skipping the test because directory type {directory_type} is not supported in region {region}") - head_node_instance_type = "c5.xlarge" - compute_instance_type_info = {"name": "c5.xlarge", "num_cores": 4} fsx_supported = is_fsx_supported(region) config_params = { - "compute_instance_type": compute_instance_type_info.get("name"), - "head_node_instance_type": head_node_instance_type, "fsx_supported": fsx_supported, } directory_stack_name, nlb_stack_name = directory_factory( @@ -826,12 +822,6 @@ def test_ad_integration_on_login_nodes( 2. SSH key for AD users is created when the property GenerateSshKeysForUsers is true; 3. AD users can submit workloads; """ - head_node_instance_type = "c5.xlarge" - compute_instance_type_info = {"name": "c5.xlarge", "num_cores": 4} - config_params = { - "compute_instance_type": compute_instance_type_info.get("name"), - "head_node_instance_type": head_node_instance_type, - } directory_stack_name, nlb_stack_name = directory_factory( request.config.getoption("directory_stack_name"), request.config.getoption("ldaps_nlb_stack_name"), @@ -846,16 +836,14 @@ def test_ad_integration_on_login_nodes( ) nlb_stack_parameters = get_infra_stack_parameters(nlb_stack_name) ldap_tls_ca_cert = "/opt/parallelcluster/shared_login_nodes/directory_service/certificate.crt" - config_params.update( - get_ad_config_param_vals( - directory_stack_outputs, - nlb_stack_parameters, - password_secret_arn, - ldap_tls_ca_cert, - directory_type, - directory_protocol, - directory_certificate_verification, - ) + config_params = get_ad_config_param_vals( + directory_stack_outputs, + nlb_stack_parameters, + password_secret_arn, + ldap_tls_ca_cert, + directory_type, + directory_protocol, + directory_certificate_verification, ) cluster_config = pcluster_config_reader(**config_params) cluster = clusters_factory(cluster_config) diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update.yaml b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update.yaml index ccd74f35da..a3c1221995 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update.yaml +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update.yaml @@ -1,7 +1,7 @@ Image: Os: {{ os }} HeadNode: - InstanceType: {{ head_node_instance_type }} + InstanceType: {{ instance }} Networking: SubnetId: {{ public_subnet_id }} Ssh: @@ -15,7 +15,7 @@ Scheduling: ComputeResources: - Name: cit Instances: - - InstanceType: {{ compute_instance_type }} + - InstanceType: {{ instance }} MinCount: 2 MaxCount: 150 Networking: diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update2.yaml b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update2.yaml index 67e7d0e517..a981e22334 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update2.yaml +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.update2.yaml @@ -1,7 +1,7 @@ Image: Os: {{ os }} HeadNode: - InstanceType: {{ head_node_instance_type }} + InstanceType: {{ instance }} Networking: SubnetId: {{ public_subnet_id }} Ssh: @@ -15,7 +15,7 @@ Scheduling: ComputeResources: - Name: cit Instances: - - InstanceType: {{ compute_instance_type }} + - InstanceType: {{ instance }} MinCount: 2 MaxCount: 150 Networking: diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.yaml b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.yaml index 70865eabea..1e2161d7f5 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.yaml +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration/pcluster.config.yaml @@ -1,7 +1,7 @@ Image: Os: {{ os }} HeadNode: - InstanceType: {{ head_node_instance_type }} + InstanceType: {{ instance }} Networking: SubnetId: {{ public_subnet_id }} Ssh: @@ -15,7 +15,7 @@ Scheduling: ComputeResources: - Name: cit Instances: - - InstanceType: {{ compute_instance_type }} + - InstanceType: {{ instance }} MinCount: 2 MaxCount: 150 Networking: diff --git a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration_on_login_nodes/pcluster.config.yaml b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration_on_login_nodes/pcluster.config.yaml index 444bea1157..f6bb9de49f 100644 --- a/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration_on_login_nodes/pcluster.config.yaml +++ b/tests/integration-tests/tests/ad_integration/test_ad_integration/test_ad_integration_on_login_nodes/pcluster.config.yaml @@ -11,7 +11,7 @@ LoginNodes: Ssh: KeyName: {{ key_name }} HeadNode: - InstanceType: {{ head_node_instance_type }} + InstanceType: {{ instance }} Networking: SubnetId: {{ public_subnet_id }} Ssh: @@ -25,7 +25,7 @@ Scheduling: ComputeResources: - Name: cit Instances: - - InstanceType: {{ compute_instance_type }} + - InstanceType: {{ instance }} MinCount: 2 MaxCount: 150 Networking: diff --git a/tests/integration-tests/tests/efa/test_efa.py b/tests/integration-tests/tests/efa/test_efa.py index 7f5bb2ec69..bf10dd193e 100644 --- a/tests/integration-tests/tests/efa/test_efa.py +++ b/tests/integration-tests/tests/efa/test_efa.py @@ -39,16 +39,13 @@ def test_efa( Grouped all tests in a single function so that cluster can be reused for all of them. """ - - max_queue_size = 2 - if architecture == "x86_64": head_node_instance = "c5.18xlarge" else: head_node_instance = "c6g.16xlarge" slots_per_instance = fetch_instance_slots(region, instance, multithreading_disabled=True) - cluster_config = pcluster_config_reader(max_queue_size=max_queue_size, head_node_instance=head_node_instance) + cluster_config = pcluster_config_reader(head_node_instance=head_node_instance) cluster = clusters_factory(cluster_config) remote_command_executor = RemoteCommandExecutor(cluster) scheduler_commands = scheduler_commands_factory(remote_command_executor) diff --git a/tests/integration-tests/tests/efa/test_efa/test_efa/pcluster.config.yaml b/tests/integration-tests/tests/efa/test_efa/test_efa/pcluster.config.yaml index 0d438c4232..7435eac99e 100644 --- a/tests/integration-tests/tests/efa/test_efa/test_efa/pcluster.config.yaml +++ b/tests/integration-tests/tests/efa/test_efa/test_efa/pcluster.config.yaml @@ -27,8 +27,8 @@ Scheduling: Instances: - InstanceType: {{ instance }} {% endif %} - MaxCount: {{ max_queue_size }} - MinCount: {{ max_queue_size }} + MaxCount: 2 + MinCount: 2 DisableSimultaneousMultithreading: true Efa: Enabled: true diff --git a/tests/integration-tests/tests/performance_tests/test_osu.py b/tests/integration-tests/tests/performance_tests/test_osu.py index 205a063baf..9265ec9cfe 100644 --- a/tests/integration-tests/tests/performance_tests/test_osu.py +++ b/tests/integration-tests/tests/performance_tests/test_osu.py @@ -1,4 +1,4 @@ -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). # You may not use this file except in compliance with the License. @@ -21,6 +21,9 @@ from tests.common.osu_common import run_individual_osu_benchmark from tests.common.utils import fetch_instance_slots, get_installed_parallelcluster_version, run_system_analyzer +# We collected OSU benchmarks results for c5n.18xlarge only. +OSU_BENCHMARKS_INSTANCES = ["c5n.18xlarge"] + @pytest.mark.usefixtures("serial_execution_by_instance") def test_osu( @@ -37,14 +40,11 @@ def test_osu( scheduler_commands_factory, request, ): - # We collected OSU benchmarks results for c5n.18xlarge only. - osu_benchmarks_instances = ["c5n.18xlarge"] - - if instance not in osu_benchmarks_instances: - raise Exception(f"OSU benchmarks can't be run on instance {instance}.") - - # 32 instances are required to see performance differences in collective OSU benchmarks. - max_queue_size = 32 + if instance not in OSU_BENCHMARKS_INSTANCES: + raise Exception( + f"OSU benchmarks can't be run on instance {instance}. " + f"Only these instances are supported: {OSU_BENCHMARKS_INSTANCES}" + ) if architecture == "x86_64": head_node_instance = "c5.18xlarge" @@ -52,7 +52,7 @@ def test_osu( head_node_instance = "c6g.16xlarge" slots_per_instance = fetch_instance_slots(region, instance, multithreading_disabled=True) - cluster_config = pcluster_config_reader(max_queue_size=max_queue_size, head_node_instance=head_node_instance) + cluster_config = pcluster_config_reader(head_node_instance=head_node_instance) cluster = clusters_factory(cluster_config) remote_command_executor = RemoteCommandExecutor(cluster) scheduler_commands = scheduler_commands_factory(remote_command_executor) @@ -83,7 +83,7 @@ def test_osu( test_datadir, os, instance, - num_instances=max_queue_size, + num_instances=32, slots_per_instance=slots_per_instance, partition="efa-enabled", ) diff --git a/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml index 36562e4149..e3113a5c0e 100644 --- a/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml +++ b/tests/integration-tests/tests/performance_tests/test_osu/test_osu/pcluster.config.yaml @@ -26,8 +26,8 @@ Scheduling: Instances: - InstanceType: {{ instance }} {% endif %} - MaxCount: {{ max_queue_size }} - MinCount: {{ max_queue_size }} + MaxCount: 32 + MinCount: 32 DisableSimultaneousMultithreading: true Efa: Enabled: true