Sync develop to stable

.github/workflows/dependencies-macos.sh

@@ -4,6 +4,8 @@ set -eu -o pipefail
 
 brew install open-mpi automake m4 libtool
 
+export CMAKE_POLICY_VERSION_MINIMUM=3.5
+
 # libfabric
 wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.15.2.tar.gz
 tar xf v1.15.2.tar.gz
@@ -19,6 +21,6 @@ git clone --recursive https://github.com/mercury-hpc/mercury.git
 cd mercury
 git checkout v2.2.0
 mkdir build && cd build
-cmake ../  -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF -DNA_OFI_TESTING_PROTOCOL=sockets
+cmake ../  -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF -DNA_OFI_TESTING_PROTOCOL=sockets -DCMAKE_POLICY_VERSION_MINIMUM=3.5
 make -j2 && sudo make install
 ctest

.github/workflows/gitlab-stable.yml

@@ -22,7 +22,7 @@ jobs:
 
       - name: Update branch
         run: |
-          git fetch stable
+          git fetch origin stable
           git remote add gitlab https://${{ secrets.GITLAB_TOKEN_NAME }}:${{ secrets.GITLAB_TOKEN }}@${{ secrets.GITLAB_URL }}
           git checkout stable
           git push -f gitlab -u stable

.github/workflows/macos.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Build PDC
         run: |
           mkdir build && cd build
-          cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc
+          cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DPDC_ENABLE_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_POLICY_VERSION_MINIMUM=3.5
           make -j 2
 
       - name: Test PDC

.github/workflows/nersc.yml

@@ -1,15 +1,17 @@
 name: NERSC
 
 on:
-  pull_request_target
+  pull_request
 
 env:
   PR_NUMBER: ${{ github.event.number }}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   authorize:
-    environment:
-      ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }}
     runs-on: ubuntu-latest
     steps:
       - run: true
@@ -35,4 +37,4 @@ jobs:
           git fetch origin pull/${PR_NUMBER}/head:PR-${PR_SHA}
           git remote add gitlab https://${{ secrets.GITLAB_TOKEN_NAME }}:${{ secrets.GITLAB_TOKEN }}@${{ secrets.GITLAB_URL }}
           git checkout PR-${PR_SHA}
-          git push -f gitlab -u PR-${PR_SHA}
+          git push -f gitlab -u PR-${PR_SHA}

.github/workflows/store-metrics.py

@@ -92,52 +92,52 @@
 
     for line in lines:
         if 'Obj create time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             obj_create_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             obj_create_time_node.append(t)
         elif 'Transfer create time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             xfer_create_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             xfer_create_time_node.append(t)                     
         elif 'Transfer start time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             xfer_start_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             xfer_start_time_node.append(t)
         elif 'Transfer wait time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             xfer_wait_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             xfer_wait_time_node.append(t)        
         elif 'Transfer close time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             xfer_close_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             xfer_close_time_node.append(t)
         elif 'Obj close time' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             obj_close_time_file_agg += t
             total_time_file_agg += t
             observed_time_file_agg += t
             obj_close_time_node.append(t)
         elif 'Sleep end' in line:
-            t = float(line.split(':')[1])
+            t = float(line.split(':')[-1])
             sleep_time_file_agg += t
             observed_time_file_agg += t
             sleep_time_node.append(t)
 
         if 'PDC Metadata servers, running with' in line and (pdc_metadata_servers is None or pdc_metadata_clients is None):
             parts = line.split()
 
-            pdc_metadata_servers = parts[2]
-            pdc_metadata_clients = parts[8]
+            pdc_metadata_servers = parts[3]
+            pdc_metadata_clients = parts[9]
 
     obj_create_time_file.append(obj_create_time_file_agg)
     obj_close_time_file.append(obj_close_time_file_agg)
@@ -270,7 +270,7 @@
             'xfer_wait_time_step': xfer_wait_time_node[step],
             'xfer_close_time_step': xfer_close_time_node[step],
             'obj_close_time_step': obj_close_time_node[step],
-            'sleep_time_step': sleep_time_node[step] if len(sleep_time_node) < step else 0
+            'sleep_time_step': sleep_time_node[step] if step < len(sleep_time_node) else 0
         }
 
         lines.append(line)

.gitlab-ci.yml

@@ -16,13 +16,13 @@ stages:
         # For complete details on the GitHub API please see:
         # https://developer.github.com/v3/repos/statuses
         - |
-          curl -L \
-          -X POST \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer ${GITHUB_TOKEN}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \
-          -d "{\"state\":\"${CI_JOB_NAME}\",\"description\":\"${STATUS_MESSAGE}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}"
+            curl -L \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \
+            -d "{\"state\":\"${CI_JOB_NAME}\",\"description\":\"${STATUS_MESSAGE}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}"
 
 pending:
     stage: .pre
@@ -50,20 +50,22 @@ failure:
 perlmutter-authorize:
     stage: authorize
     rules:
-    - if: '$METRICS == null'
-      when: manual
+        - if: '$METRICS == null'
+    when: manual
     script:
         - |
-          curl -L \
-          -X POST \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer ${GITHUB_TOKEN}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \
-          -d "{\"state\":\"pending\",\"description\":\"Running...\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}"
+            curl -L \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" \
+            -d "{\"state\":\"pending\",\"description\":\"Running...\",\"target_url\":\"${CI_PIPELINE_URL}\",\"context\":\"${STATUS_NAME}\"}"
 
 perlmutter-no-cache-build:
     stage: build
+    rules:
+        - if: '$METRICS == null'
     needs:
         - perlmutter-authorize
     id_tokens:
@@ -91,11 +93,13 @@ perlmutter-no-cache-build:
 
 perlmutter-cache-build:
     stage: build
-    needs:
-        - perlmutter-authorize
     id_tokens:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
+    rules:
+        - if: '$METRICS == null'
+    needs:
+        - perlmutter-authorize
     allow_failure: false
     tags:
         - perlmutter
@@ -127,7 +131,7 @@ perlmutter-no-cache-parallel-pdc:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
     tags:
@@ -150,7 +154,7 @@ perlmutter-no-cache-parallel-obj:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
         - perlmutter-no-cache-parallel-pdc
@@ -174,7 +178,7 @@ perlmutter-no-cache-parallel-cont:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
         - perlmutter-no-cache-parallel-pdc
@@ -198,7 +202,7 @@ perlmutter-no-cache-parallel-prop:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
         - perlmutter-no-cache-parallel-pdc
@@ -222,7 +226,7 @@ perlmutter-no-cache-parallel-region:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
         - perlmutter-no-cache-parallel-pdc
@@ -246,7 +250,7 @@ perlmutter-no-cache-parallel-region-all:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-no-cache-build
         - perlmutter-no-cache-parallel-pdc
@@ -275,9 +279,10 @@ perlmutter-cache-parallel-pdc:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
+    allow_failure: false
     tags:
         - perlmutter
     variables:
@@ -298,7 +303,7 @@ perlmutter-cache-parallel-obj:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
         - perlmutter-cache-parallel-pdc
@@ -322,7 +327,7 @@ perlmutter-cache-parallel-cont:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
         - perlmutter-cache-parallel-pdc
@@ -346,7 +351,7 @@ perlmutter-cache-parallel-prop:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
         - perlmutter-cache-parallel-pdc
@@ -370,7 +375,7 @@ perlmutter-cache-parallel-region:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
         - perlmutter-cache-parallel-pdc
@@ -394,7 +399,7 @@ perlmutter-cache-parallel-region-all:
         SITE_ID_TOKEN:
             aud: https://gitlab.com/
     rules:
-    - if: '$METRICS == null'
+        - if: '$METRICS == null'
     needs:
         - perlmutter-cache-build
         - perlmutter-cache-parallel-pdc