Add CUDA-aware MPI flags to QE build

src/qe.py

@@ -39,7 +39,8 @@ def build(self):
         runtime, cuda_cc = device_query(self.host[0])
 
         self.check_prerequisite('hpc_sdk', '21.5')
-
+
+        # test 
         self.buildcmd += [
            f'wget https://gitlab.com/QEF/q-e/-/archive/develop/q-e-develop.tar.gz -O {self.builddir}/q-e-develop.tar.gz', 
            f'cd {self.builddir}; tar xf q-e-develop.tar.gz',
@@ -48,9 +49,10 @@ def build(self):
                 '--enable-openmp '
                f'--with-cuda={os.environ["NVHPC_ROOT"]}/cuda/{runtime} '
                f'--with-cuda-cc={cuda_cc} '
-               f'--with-cuda-runtime={runtime} '
-            'make -j 8 pw;' 
-            'make -j 8 neb;'
+               f'--with-cuda-runtime={runtime}; '
+            'perl -pi -e "s/^(DFLAGS.*)/\$1 -D__GPU_MPI/" make.inc; '
+            'make -j 8 pw; ' 
+            'make -j 8 neb; '
             'make install')]
 
         super().build() 

test/test_stream_omp.py

@@ -10,7 +10,7 @@
 
 # scan affinity/thread 
 for affinity in ['close', 'spread']: 
-    for thread in [1, 2, 4, 8, 16, 24, 32]: 
+    for thread in [1, 2, 4, 8, 16, 24, 32, 40]: 
         stream.affinity = affinity
         stream.thread   = thread