From feee53b86b8b4304b39c689bc5353381e9178757 Mon Sep 17 00:00:00 2001 From: Bruce J Palmer Date: Wed, 30 Oct 2024 13:27:20 -0700 Subject: [PATCH] Modified PR runtime to use hostnames instead of host IDs to keep track of SMP nodes. --- comex/src-mpi-pr/comex.c | 36 +++++++++++++++---------------- comex/src-mpi-pr/groups.c | 45 ++++++++++++++++++++++++++++----------- comex/src-mpi-pr/groups.h | 8 ++++++- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/comex/src-mpi-pr/comex.c b/comex/src-mpi-pr/comex.c index f485798a4..ae2306efc 100644 --- a/comex/src-mpi-pr/comex.c +++ b/comex/src-mpi-pr/comex.c @@ -2722,8 +2722,8 @@ int comex_malloc_mem_dev(void *ptrs[], size_t size, comex_group_t group, reg_entries_local[reg_entries_local_count++] = reg_entries[i]; } } - else if (g_state.hostid[reg_entries[i].rank] - == g_state.hostid[my_world_rank]) { + else if (!strcmp(g_state.host[reg_entries[i].rank].name, + g_state.host[my_world_rank].name)) { /* same SMP node, need to mmap */ /* open remote shared memory object */ void *memory = _shm_attach_memdev(reg_entries[i].name, @@ -2880,7 +2880,7 @@ void _malloc_semaphore() if (g_state.rank == i) { continue; /* skip my own rank */ } - else if (g_state.hostid[g_state.rank] == g_state.hostid[i]) { + else if (!strcmp(g_state.host[g_state.rank].name,g_state.host[i].name)) { /* same SMP node */ #if ENABLE_UNNAMED_SEM semaphores[i] = _shm_attach( @@ -2949,7 +2949,7 @@ void _free_semaphore() } #endif } - else if (g_state.hostid[g_state.rank] == g_state.hostid[i]) { + else if (!strcmp(g_state.host[g_state.rank].name,g_state.host[i].name)) { /* same SMP node */ #if ENABLE_UNNAMED_SEM retval = munmap(semaphores[i], sizeof(sem_t)); @@ -4648,8 +4648,8 @@ STATIC void _malloc_handler( fprintf(stderr, "[%d] _malloc_handler found NULL at %d\n", g_state.rank, i); #endif } - else if (g_state.hostid[reg_entries[i].rank] - == g_state.hostid[g_state.rank]) { + else if (!strcmp(g_state.host[reg_entries[i].rank].name, + g_state.host[g_state.rank].name)) { /* same SMP node, need to mmap */ /* attach to remote shared memory object */ void *memory; @@ -4741,8 +4741,8 @@ STATIC void _free_handler(header_t *header, char *payload, int proc) fprintf(stderr, "[%d] _free_handler found NULL at %d\n", g_state.rank, i); #endif } - else if (g_state.hostid[rank_ptrs[i].rank] - == g_state.hostid[g_state.rank]) { + else if (!strcmp(g_state.host[rank_ptrs[i].rank].name, + g_state.host[g_state.rank].name)) { /* same SMP node */ reg_entry_t *reg_entry = NULL; int retval = 0; @@ -4926,7 +4926,7 @@ STATIC int _smallest_world_rank_with_same_hostid(comex_igroup_t *igroup) int *world_ranks = _get_world_ranks(igroup); for (i=0; isize; ++i) { - if (g_state.hostid[world_ranks[i]] == g_state.hostid[g_state.rank]) { + if (!strcmp(g_state.host[world_ranks[i]].name,g_state.host[g_state.rank].name)) { /* found same host as me */ if (world_ranks[i] < smallest) { smallest = world_ranks[i]; @@ -4949,7 +4949,7 @@ STATIC int _largest_world_rank_with_same_hostid(comex_igroup_t *igroup) int *world_ranks = _get_world_ranks(igroup); for (i=0; isize; ++i) { - if (g_state.hostid[world_ranks[i]] == g_state.hostid[g_state.rank]) { + if (!strcmp(g_state.host[world_ranks[i]].name,g_state.host[g_state.rank].name)) { /* found same host as me */ if (world_ranks[i] > largest) { largest = world_ranks[i]; @@ -6365,7 +6365,7 @@ STATIC void nb_puts( if (COMEX_ENABLE_PUT_DATATYPE && (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc) && (!COMEX_ENABLE_PUT_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank]) + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name)) && (_packed_size(src_stride, count, stride_levels) > COMEX_PUT_DATATYPE_THRESHOLD)) { nb_puts_datatype(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb); return; @@ -6375,7 +6375,7 @@ STATIC void nb_puts( if (COMEX_ENABLE_PUT_PACKED && (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc) && (!COMEX_ENABLE_PUT_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_puts_packed(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb); return; } @@ -6634,7 +6634,7 @@ STATIC void nb_gets( if (COMEX_ENABLE_GET_DATATYPE && (!COMEX_ENABLE_GET_SELF || g_state.rank != proc) && (!COMEX_ENABLE_GET_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank]) + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name)) && (_packed_size(src_stride, count, stride_levels) > COMEX_GET_DATATYPE_THRESHOLD)) { nb_gets_datatype(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb); return; @@ -6644,7 +6644,7 @@ STATIC void nb_gets( if (COMEX_ENABLE_GET_PACKED && (!COMEX_ENABLE_GET_SELF || g_state.rank != proc) && (!COMEX_ENABLE_GET_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_gets_packed(src, src_stride, dst, dst_stride, count, stride_levels, proc, nb); return; } @@ -6910,7 +6910,7 @@ STATIC void nb_accs( if (COMEX_ENABLE_ACC_PACKED && (!COMEX_ENABLE_ACC_SELF || g_state.rank != proc) && (!COMEX_ENABLE_ACC_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_accs_packed(datatype, scale, src, src_stride, dst, dst_stride, count, stride_levels, proc, nb); return; } @@ -7120,7 +7120,7 @@ STATIC void nb_putv( if (COMEX_ENABLE_PUT_IOV && (!COMEX_ENABLE_PUT_SELF || g_state.rank != proc) && (!COMEX_ENABLE_PUT_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_putv_packed(&iov[i], proc, nb); } else { @@ -7224,7 +7224,7 @@ STATIC void nb_getv( if (COMEX_ENABLE_GET_IOV && (!COMEX_ENABLE_GET_SELF || g_state.rank != proc) && (!COMEX_ENABLE_GET_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_getv_packed(&iov[i], proc, nb); } else { @@ -7336,7 +7336,7 @@ STATIC void nb_accv( if (COMEX_ENABLE_ACC_IOV && (!COMEX_ENABLE_ACC_SELF || g_state.rank != proc) && (!COMEX_ENABLE_ACC_SMP - || g_state.hostid[proc] != g_state.hostid[g_state.rank])) { + || strcmp(g_state.host[proc].name,g_state.host[g_state.rank].name))) { nb_accv_packed(datatype, scale, &iov[i], proc, nb); } else { diff --git a/comex/src-mpi-pr/groups.c b/comex/src-mpi-pr/groups.c index aa49233cc..6ecbf73a1 100644 --- a/comex/src-mpi-pr/groups.c +++ b/comex/src-mpi-pr/groups.c @@ -381,6 +381,27 @@ static int cmplong(const void *p1, const void *p2) return *((long*)p1) - *((long*)p2); } +static int cmpname(const void *name1, const void *name2) +{ + const char* n1 = (const char*)name1; + const char* n2 = (const char*)name2; + int comp = 0; + int i; + for (i=0; i (int)n2[i]) { + comp = 1; + break; + } else if (n1[i] == '\0' || n2[i] == '\0') { + break; + } + } + return comp; +} + + /** * Initialize group linked list. Prepopulate with world group. */ @@ -393,7 +414,7 @@ void comex_group_init(MPI_Comm comm) int size_node = 0; comex_group_t group = 0; comex_igroup_t *igroup = NULL; - long *sorted = NULL; + host_name_t *sorted = NULL; int count = 0; /* populate g_state */ @@ -420,10 +441,10 @@ void comex_group_init(MPI_Comm comm) #endif /* need to figure out which proc is master on each node */ - g_state.hostid = (long*)malloc(sizeof(long)*g_state.size); - g_state.hostid[g_state.rank] = xgethostid(); - status = MPI_Allgather(MPI_IN_PLACE, 1, MPI_LONG, - g_state.hostid, 1, MPI_LONG, g_state.comm); + g_state.host = (host_name_t*)malloc(sizeof(host_name_t)*g_state.size); + gethostname(g_state.host[g_state.rank].name,COMEX_MAX_HOST_NAME_LEN); + status = MPI_Allgather(MPI_IN_PLACE, sizeof(host_name_t), MPI_BYTE, + g_state.host, sizeof(host_name_t), MPI_BYTE, g_state.comm); COMEX_ASSERT(MPI_SUCCESS == status); /* First create a temporary node communicator and then * split further into number of groups within the node */ @@ -432,17 +453,17 @@ void comex_group_init(MPI_Comm comm) /* create node comm */ /* MPI_Comm_split requires a non-negative color, * so sort and sanitize */ - sorted = (long*)malloc(sizeof(long) * g_state.size); - (void)memcpy(sorted, g_state.hostid, sizeof(long)*g_state.size); - qsort(sorted, g_state.size, sizeof(long), cmplong); + sorted = (long*)malloc(sizeof(host_name_t) * g_state.size); + (void)memcpy(sorted, g_state.host, sizeof(host_name_t)*g_state.size); + qsort(sorted, g_state.size, sizeof(host_name_t), cmpname); /* count is number of distinct host IDs that are lower than * the host ID of this rank */ for (i=0; i #include "comex.h" +typedef struct { + char name[COMEX_MAX_HOST_NAME_LEN]; +} host_name_t; + typedef struct { MPI_Comm comm; /**< whole comm; all ranks */ MPI_Group group;/**< whole group; all ranks */ int size; /**< comm size */ int rank; /**< comm rank */ int *master; /**< master[size] rank of a given rank's master */ - long *hostid; /**< hostid[size] hostid of SMP node for a given rank */ + host_name_t *host; /**< host[size] host name of SMP node for a given rank */ MPI_Comm node_comm; /**< node comm; SMP ranks */ int node_size; /**< node comm size */ int node_rank; /**< node comm rank */