diff --git a/ASOFI3D/mfiles/snap3D_allplanes.m b/ASOFI3D/mfiles/snap3D_allplanes.m index a19ea0f..89d6142 100644 --- a/ASOFI3D/mfiles/snap3D_allplanes.m +++ b/ASOFI3D/mfiles/snap3D_allplanes.m @@ -73,7 +73,7 @@ TSNAPINC=0.2; % firts and last snapshot that is considered for displayin firstframe=1; -lastframe=20; +lastframe=60; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %---3D definitions: defines two rotating planes (xz, yz plane) diff --git a/ASOFI3D/par/compileSOFI3D.sh b/ASOFI3D/par/compileSOFI3D.sh index c784b5e..1c8d6b5 100755 --- a/ASOFI3D/par/compileSOFI3D.sh +++ b/ASOFI3D/par/compileSOFI3D.sh @@ -1,8 +1,16 @@ #---- general compilation cd ../src -make +#make sofi3D + +make clean +make -j 8 sofi3D + #make clean -# make sofi3D +# make sofi3D + #make snapmerge cd ../par + + + diff --git a/ASOFI3D/par/in_and_out/sofi3D.json b/ASOFI3D/par/in_and_out/sofi3D.json index 7d3c089..c053c76 100644 --- a/ASOFI3D/par/in_and_out/sofi3D.json +++ b/ASOFI3D/par/in_and_out/sofi3D.json @@ -7,7 +7,7 @@ { "Imaging" : "comment", - "RTM_FLAG" : "0", + "RTM_FLAG" : "1", "Domain Decomposition" : "comment", "NPROCX" : "2", @@ -31,7 +31,7 @@ "fdcoeff values: Taylor=1, Holberg=2" : "comment", "Time Stepping" : "comment", - "TIME" : "5.0", + "TIME" : "5", "DT" : "6.6e-3", "Source" : "comment", @@ -79,9 +79,9 @@ "Snapshots" : "comment", "SNAP" : "4", - "TSNAP1" : "0.05", - "TSNAP2" : "1.0", - "TSNAPINC" : "0.05", + "TSNAP1" : "6.6e-3", + "TSNAP2" : "9.6", + "TSNAPINC" : "0.2", "IDX" : "1", "IDY" : "1", "IDZ" : "1", @@ -90,7 +90,7 @@ "SNAP_PLANE" : "2", "Receiver" : "comment", - "SEISMO" : "4", + "SEISMO" : "1", "READREC" : "0", "REC_FILE" : "./receiver/receiver.dat", "REFRECX, REFRECY, REFRECZ" : "0.0 , 0.0 , 0.0", @@ -115,7 +115,7 @@ "LOG_FILE" : "log/test.log", "LOG" : "1", "OUT_SOURCE_WAVELET" : "1", - "OUT_TIMESTEP_INFO" : "250", + "OUT_TIMESTEP_INFO" : "50", "Checkpoints" : "comment", "CHECKPTREAD" : "0", @@ -125,4 +125,5 @@ "Madagascar" : "comment", "RSF" : "1", "RSFDEN" : "./madagascar/test_rho.rsf", + } diff --git a/ASOFI3D/par/madagascar/test_den1.rsf b/ASOFI3D/par/madagascar/test_den1.rsf deleted file mode 100644 index 60f4df9..0000000 --- a/ASOFI3D/par/madagascar/test_den1.rsf +++ /dev/null @@ -1,17 +0,0 @@ -1.6-svn sfmath ASOFI3D/par/madagascar: kalim0a@kw12159 Thu Dec 22 14:00:42 2016 - - x2=1 - d2=54 - n2=40 - o1=0 - x3=2 - d3=54 - n3=60 - o2=0 - data_format="native_float" - o3=0 - esize=4 - in="/home/kalim0a/RSFTMP/ASOFI3D/par/madagascar/test_den1.rsf@" - x1=0 - d1=54 - n1=60 diff --git a/ASOFI3D/par/madagascar/test_den2.rsf b/ASOFI3D/par/madagascar/test_den2.rsf deleted file mode 100644 index 21534f6..0000000 --- a/ASOFI3D/par/madagascar/test_den2.rsf +++ /dev/null @@ -1,17 +0,0 @@ -1.6-svn sfmath ASOFI3D/par/madagascar: kalim0a@kw12159 Thu Dec 22 14:00:42 2016 - - x2=1 - d2=54 - n2=20 - o1=0 - x3=2 - d3=54 - n3=60 - o2=0 - data_format="native_float" - o3=0 - esize=4 - in="/home/kalim0a/RSFTMP/ASOFI3D/par/madagascar/test_den2.rsf@" - x1=0 - d1=54 - n1=60 diff --git a/ASOFI3D/par/madagascar/test_rho.rsf b/ASOFI3D/par/madagascar/test_rho.rsf deleted file mode 100644 index 5bbcc2c..0000000 --- a/ASOFI3D/par/madagascar/test_rho.rsf +++ /dev/null @@ -1,37 +0,0 @@ -1.6-svn sfmath ASOFI3D/par/madagascar: kalim0a@kw12159 Thu Dec 22 14:00:42 2016 - - x2=1 - d2=54 - n2=40 - o1=0 - x3=2 - d3=54 - n3=60 - o2=0 - data_format="native_float" - o3=0 - esize=4 - in="/home/kalim0a/RSFTMP/ASOFI3D/par/madagascar/test_den1.rsf@" - x1=0 - d1=54 - n1=60 -1.6-svn sfcat ASOFI3D/par/madagascar: kalim0a@kw12159 Thu Dec 22 14:00:42 2016 - - d2=54 - n2=60 - o2=0 - data_format="native_float" - esize=4 - in="stdout" - in="stdin" - -1.6-svn sfput ASOFI3D/par/madagascar: kalim0a@kw12159 Thu Dec 22 14:00:42 2016 - - unit2=m - label3=z - unit3=m - data_format="native_float" - in="/home/kalim0a/RSFTMP/ASOFI3D/par/madagascar/test_rho.rsf@" - label1=y - unit1=m - label2=x diff --git a/ASOFI3D/par/nvprof.1 b/ASOFI3D/par/nvprof.1 new file mode 100644 index 0000000..fc9b637 --- /dev/null +++ b/ASOFI3D/par/nvprof.1 @@ -0,0 +1,61 @@ +==45842== Profiling result: +Time(%) Time Calls Avg Min Max Name + 25.86% 15.4165s 76 202.85ms 198.15ms 205.01ms update_s_elastic_159_gpu + 12.69% 7.56531s 76 99.544ms 98.223ms 100.82ms update_v_126_gpu + 9.41% 5.60882s 76 73.800ms 69.868ms 81.132ms update_s_CPML_elastic_591_gpu + 7.49% 4.46770s 76 58.786ms 48.310ms 80.522ms update_s_CPML_elastic_479_gpu + 6.84% 4.07530s 76 53.622ms 53.373ms 53.846ms update_s_CPML_elastic_393_gpu + 6.81% 4.06096s 76 53.434ms 52.938ms 53.937ms update_s_CPML_elastic_311_gpu + 5.10% 3.04203s 76 40.027ms 39.876ms 40.315ms update_v_CPML_407_gpu + 4.92% 2.93369s 76 38.601ms 38.386ms 38.838ms update_v_CPML_314_gpu + 4.83% 2.87799s 76 37.868ms 37.593ms 38.236ms update_v_CPML_258_gpu + 3.54% 2.11239s 76 27.795ms 25.102ms 40.658ms update_v_CPML_371_gpu + 3.29% 1.96067s 76 25.798ms 25.554ms 26.036ms update_s_CPML_elastic_98_gpu + 3.26% 1.94498s 76 25.592ms 25.434ms 25.719ms update_s_CPML_elastic_202_gpu + 2.34% 1.39695s 76 18.381ms 18.205ms 18.545ms update_v_CPML_105_gpu + 2.32% 1.38053s 76 18.165ms 18.065ms 18.257ms update_v_CPML_181_gpu + 0.65% 390.27ms 10704 36.459us 4.4480us 67.648us [CUDA memcpy DtoH] + 0.64% 381.92ms 1736 220.00us 3.0720us 1.9478ms [CUDA memcpy HtoD] + 0.00% 833.16us 6 138.86us 135.97us 146.79us [CUDA memcpy DtoD] + +==43017== Profiling application: ../bin/sofi3D ./in_and_out/sofi3D.json +==43017== Profiling result: +Time(%) Time Calls Avg Min Max Name + 26.40% 15.4008s 76 202.64ms 198.09ms 205.20ms update_s_elastic_159_gpu + 12.95% 7.55518s 76 99.410ms 98.122ms 100.59ms update_v_126_gpu + 9.58% 5.58590s 76 73.499ms 70.073ms 81.162ms update_s_CPML_elastic_588_gpu + 7.63% 4.45298s 76 58.592ms 48.052ms 80.914ms update_s_CPML_elastic_476_gpu + 6.98% 4.07094s 76 53.565ms 53.331ms 53.780ms update_s_CPML_elastic_390_gpu + 6.95% 4.05447s 76 53.348ms 52.808ms 53.755ms update_s_CPML_elastic_308_gpu + 5.21% 3.03771s 76 39.970ms 39.731ms 40.178ms update_v_CPML_407_gpu + 5.03% 2.93152s 76 38.573ms 38.361ms 38.887ms update_v_CPML_314_gpu + 4.93% 2.87687s 76 37.854ms 37.597ms 38.148ms update_v_CPML_258_gpu + 3.63% 2.11601s 76 27.842ms 25.033ms 40.485ms update_v_CPML_371_gpu + 3.33% 1.94405s 76 25.580ms 25.464ms 25.723ms update_s_CPML_elastic_199_gpu + 2.39% 1.39591s 76 18.367ms 18.213ms 18.519ms update_v_CPML_105_gpu + 2.36% 1.37828s 76 18.135ms 18.036ms 18.252ms update_v_CPML_181_gpu + 1.35% 786.87ms 76 10.354ms 10.282ms 10.418ms update_s_CPML_elastic_98_gpu + 0.65% 379.43ms 10704 35.447us 4.4480us 51.232us [CUDA memcpy DtoH] + 0.63% 369.39ms 1736 212.78us 3.0400us 942.06us [CUDA memcpy HtoD] + 0.00% 828.32us 6 138.05us 136.54us 140.48us [CUDA memcpy DtoD] + +==126591== Profiling result: +Time(%) Time Calls Avg Min Max Name + 13.24% 6.31063s 76 83.035ms 82.987ms 83.086ms update_s_elastic_159_gpu + 12.30% 5.86413s 76 77.160ms 70.381ms 82.308ms update_s_CPML_elastic_589_gpu + 11.36% 5.41593s 76 71.262ms 71.229ms 71.292ms update_v_126_gpu + 9.70% 4.62594s 76 60.868ms 48.595ms 81.910ms update_s_CPML_elastic_477_gpu + 8.66% 4.12933s 76 54.333ms 54.136ms 54.514ms update_s_CPML_elastic_391_gpu + 8.53% 4.06766s 76 53.522ms 53.099ms 53.829ms update_s_CPML_elastic_309_gpu + 6.40% 3.05003s 76 40.132ms 32.902ms 40.566ms update_v_CPML_407_gpu + 6.15% 2.93372s 76 38.602ms 38.411ms 38.855ms update_v_CPML_314_gpu + 6.03% 2.87349s 76 37.809ms 37.478ms 38.172ms update_v_CPML_258_gpu + 4.45% 2.11982s 76 27.892ms 25.263ms 40.809ms update_v_CPML_371_gpu + 4.09% 1.95082s 76 25.669ms 25.567ms 25.863ms update_s_CPML_elastic_200_gpu + 2.94% 1.40140s 76 18.440ms 18.293ms 18.567ms update_v_CPML_105_gpu + 2.90% 1.38065s 76 18.166ms 18.069ms 18.215ms update_v_CPML_181_gpu + 1.66% 791.74ms 76 10.418ms 10.363ms 10.504ms update_s_CPML_elastic_98_gpu + 0.81% 384.25ms 10710 35.877us 4.5120us 46.912us [CUDA memcpy DtoH] + 0.77% 367.67ms 1739 211.43us 3.1040us 454.66us [CUDA memcpy HtoD] + 0.00% 826.37us 6 137.73us 136.10us 138.53us [CUDA memcpy DtoD] + diff --git a/ASOFI3D/par/sofi3D_1p_2017-02-07_10-17.map b/ASOFI3D/par/sofi3D_1p_2017-02-07_10-17.map new file mode 100644 index 0000000..5b30d31 Binary files /dev/null and b/ASOFI3D/par/sofi3D_1p_2017-02-07_10-17.map differ diff --git a/ASOFI3D/par/startSOFI3D.sh b/ASOFI3D/par/startSOFI3D.sh index d1b95df..205cb38 100755 --- a/ASOFI3D/par/startSOFI3D.sh +++ b/ASOFI3D/par/startSOFI3D.sh @@ -5,11 +5,12 @@ #mpirun -np 8 nice -19 ../bin/sofi3D_acoustic ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout #mpirun -np 8 nice -19 ../bin/sofi3D ./in_and_out/sofi3D.inp | tee ./in_and_out/sofi3D.out -mpirun -np $1 nice -19 ../bin/sofi3D ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout +# profiling with nvprof +#mpirun -np 8 nice -19 nvprof --cpu-profiling on ../bin/sofi3D ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout #----execute with OPENMPI2 -#mpirun -np 8 nice -19 ../bin/sofi3D ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout +mpirun -np $1 nice -19 ../bin/sofi3D ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout #mpirun --hostfile mpihosts -np 8 nice -19 ../bin/sofi3D ./in_and_out/sofi3D.json | tee ./in_and_out/sofi3D.jout #merge snapshots diff --git a/ASOFI3D/par/su/.gitignore b/ASOFI3D/par/suREF/.gitignore similarity index 100% rename from ASOFI3D/par/su/.gitignore rename to ASOFI3D/par/suREF/.gitignore diff --git a/ASOFI3D/par/su/suNimage.sh b/ASOFI3D/par/suREF/suNimage.sh similarity index 100% rename from ASOFI3D/par/su/suNimage.sh rename to ASOFI3D/par/suREF/suNimage.sh diff --git a/ASOFI3D/par/tmp.txt b/ASOFI3D/par/tmp.txt new file mode 100644 index 0000000..07e5279 --- /dev/null +++ b/ASOFI3D/par/tmp.txt @@ -0,0 +1,13 @@ + + +double *time_v_update +time_v_update = dvector(1, NT); + +double *time_s_update +time_s_update = dvector(1, NT); + +double *time_s_exchane +time_s_exchange = dvector(1, NT); + +double *time_v_exchange +time_v_exchange = dvector(1, NT); diff --git a/ASOFI3D/sofi3D_omp b/ASOFI3D/sofi3D_omp new file mode 100755 index 0000000..1072959 Binary files /dev/null and b/ASOFI3D/sofi3D_omp differ diff --git a/ASOFI3D/src/Makefile_GPU b/ASOFI3D/src/Makefile_GPU new file mode 100644 index 0000000..b944c8f --- /dev/null +++ b/ASOFI3D/src/Makefile_GPU @@ -0,0 +1,265 @@ +# Makefile for SOFI3D + +#-------------------------------------------------------- +# edit here +# modules for KAUST + +# source code for model generation (viscoelastic) +MODEL_SRC_V = hh_visco.c + +# source code for model generation (elastic) +MODEL_SRC_E = hh_elastic.c + +# source code for model generation (acoustic) +MODEL_SRC_A = hh_acoustic.c + +# source code for benchmark model +MODEL_SRC_BENCH = benchmod.c + +# Compiler (LAM: CC=hcc, CRAY T3E: CC=cc, SHARCNet: mpicc) + +# On Linux cluster running LAM +# CC=hcc +# LFLAGS=-lm -lmpi +# CFLAGS=-g -Wall -O4 + +# On CRAY T3E +# CC=cc + +# On Linux Cluster using OpenMPI + CC=mpicc +#-ta=tesla:managed -acc -Minfo=accel +# CC=mpicc -ta=multicore -acc -Minfo=accel + LFLAGS=-lm + CFLAGS=-Wall -O3 + +# On InstitutsCluster2 using intelMPI AND ITAC (no optimization) +# CC = mpicc +# LFLAGS = -tcollect +# CFLAGS = -tcollect -trace + +# On InstitutsCluster2 using Open MPI and scalasca (no optimization) +#CC = skin mpicc +#LFLAGS = +#CFLAGS =-O3 + +#On InstitutsCluster2 using Open MPI and DDT (no optimization) +#CC = mpicc +#LFLAGS = -g +#CFLAGS = -g -O0 -i_dynamic + + +# On JUROPA cluster +#CC=mpicc +#LFLAGS=-lm +#CFLAGS=-O3 -ipo + + +# On HLRN system +#CC=mpcc +#LFLAGS=-lm +# ON Linux cluster ALTIX.RZ.UNI-KIEL:DE +#CC=icc +#CFLAGS=-mp -O3 -ip0q +#LFLAGS=-lmpi -lm + +# ON Linux cluster ALTIX.HRZ.TU-FREIBERG.DE +#CC=icc +#CFLAGS=-mp -O3 -ip0 +#LFLAGS=-lmpi -lm -i-static + + +# after this line, no further editing should be necessary +# -------------------------------------------------------- + +.c.o: + $(CC) -c $(CFLAGS) $< + + +SEISMERGE_SCR = seismerge.c + + +SNAPMERGE_SCR = \ + json_parser.c\ + merge.c \ + read_par_json.c \ + readdsk.c \ + snapmerge.c \ + util.c \ + writedsk.c + + +PARTMODEL_SCR = \ + json_parser.c\ + part_model.c \ + read_par_json.c \ + util.c + + +SOFI3D_UTIL = \ + absorb.c \ + av_mat.c \ + comm_ini.c \ + catseis.c \ + info.c \ + initproc.c \ + initsour.c \ + json_parser.c\ + merge.c \ + mergemod.c \ + note.c \ + outseis.c \ + outseis_glob.c \ + output_source_signal.c \ + PML_ini.c \ + rd_sour.c \ + read_checkpoint.c\ + readdsk.c \ + read_par_json.c \ + exchange_par.c \ + receiver.c \ + save_checkpoint.c\ + saveseis.c \ + saveseis_glob.c \ + sources.c \ + splitrec.c \ + splitsrc.c \ + timing.c \ + util.c \ + wavelet.c \ + writedsk.c \ + writemod.c \ + writepar.c \ + zero_acoustic.c \ + zero.c \ + zero_elastic.c \ + zero_elastic_CPML.c \ + rwsegy.c + +SOFI3D_SRC = \ + sofi3D.c \ + comm_ini_s.c \ + checkfd_ssg.c \ + CPML_coeff.c \ + CPML_ini_elastic.c\ + eqsource.c \ + seismo_ssg.c \ + matcopy.c \ + surface_ssg.c \ + surface_ssg_elastic.c \ + update_s_ssg.c \ + update_s_ssg_elastic.c \ + update_s_ssg_CPML.c \ + update_s_ssg_CPML_elastic.c \ + update_v_ssg.c \ + update_v_ssg_CPML.c \ + snap_ssg.c \ + exchange_v.c \ + exchange_s.c \ + psource.c \ + readmod.c \ + $(MODEL_SRC_E) \ + $(MODEL_SRC_V) \ + $(SOFI3D_UTIL) \ + madinput.c \ + mad_elastic.c + +SOFI3D_SRC_BENCH = \ + sofi3D.c \ + comm_ini_s.c \ + checkfd_ssg.c \ + CPML_coeff.c \ + CPML_ini_elastic.c\ + eqsource.c \ + seismo_ssg.c \ + matcopy.c \ + surface_ssg.c \ + surface_ssg_elastic.c \ + update_s_ssg.c \ + update_s_ssg_elastic.c \ + update_s_ssg_CPML.c \ + update_s_ssg_CPML_elastic.c \ + update_v_ssg.c \ + update_v_ssg_CPML.c \ + snap_ssg.c \ + exchange_v.c \ + exchange_s.c \ + psource.c \ + readmod.c \ + $(MODEL_SRC_BENCH) \ + $(MODEL_SRC_V) \ + $(SOFI3D_UTIL) + +SOFI3D_SRC_RSG = \ + sofi3D_rsg.c \ + checkfd_rsg.c \ + seismo_rsg.c \ + update_s_rsg.c \ + matcopy.c \ + update_v_rsg.c \ + snap_rsg.c \ + exchange_v_rsg.c \ + exchange_s_rsg.c \ + psource_rsg.c \ + $(MODEL_SRC_E) \ + $(MODEL_SRC_V) \ + $(SOFI3D_UTIL) + +SOFI3D_SRC_ACOUSTIC = \ + av_mat_acoustic.c \ + sofi3D_acoustic.c \ + checkfd_acoustic.c \ + comm_ini_acoustic.c \ + seismo_ssg_acoustic.c \ + matcopy_acoustic.c \ + surface_acoustic.c \ + PML_ini_acoustic.c \ + absorb_PML.c \ + update_s_acoustic.c \ + update_v_acoustic.c \ + update_s_acoustic_PML.c \ + update_v_acoustic_PML.c \ + snap_ssg_acoustic.c \ + exchange_v_acoustic.c \ + exchange_s_acoustic.c \ + psource_acoustic.c \ + readmod_acoustic.c \ + $(MODEL_SRC_A) \ + $(SOFI3D_UTIL) + +SOFI3D_OBJ = $(SOFI3D_SRC:%.c=%.o) +SOFI3D_OBJ_RSG = $(SOFI3D_SRC_RSG:%.c=%.o) +SOFI3D_OBJ_BENCH = $(SOFI3D_SRC_BENCH:%.c=%.o) +SOFI3D_OBJ_ACOUSTIC = $(SOFI3D_SRC_ACOUSTIC:%.c=%.o) + +SNAPMERGE_OBJ = $(SNAPMERGE_SCR:%.c=%.o) +PARTMODEL_OBJ = $(PARTMODEL_SCR:%.c=%.o) +SEISMERGE_OBJ = $(SEISMERGE_SCR:%.c=%.o) + +sofi3D: $(SOFI3D_OBJ) + $(CC) $(SOFI3D_OBJ) -o ../bin/sofi3D $(LFLAGS) + +seismerge: $(SEISMERGE_OBJ) + $(CC) $(SEISMERGE_OBJ) -o ../bin/seismerge $(LFLAGS) + +snapmerge: $(SNAPMERGE_OBJ) + $(CC) $(SNAPMERGE_OBJ) -o ../bin/snapmerge $(LFLAGS) + +part_model: $(PARTMODEL_OBJ) + $(CC) $(PARTMODEL_OBJ) -o ../bin/partmodel $(LFLAGS) + +#sofi3D_rsg: $(SOFI3D_OBJ_RSG) +# $(CC) $(SOFI3D_OBJ_RSG) -o ../bin/sofi3D_rsg $(LFLAGS) + +sofi3D_bench: $(SOFI3D_OBJ_BENCH) + $(CC) $(SOFI3D_OBJ_BENCH) -o ../bin/sofi3D_bench $(LFLAGS) + +sofi3D_acoustic: $(SOFI3D_OBJ_ACOUSTIC) + $(CC) $(SOFI3D_OBJ_ACOUSTIC) -o ../bin/sofi3D_acoustic $(LFLAGS) + +clean: + find . -name "*.o" -exec rm {} \; + find . -name "*.bck" -exec rm {} \; + find . -name "core" -exec rm {} \; + +all: clean sofi3D sofi3D_acoustic snapmerge seismerge part_model diff --git a/ASOFI3D/src/sofi3D.c b/ASOFI3D/src/sofi3D.c index 0c14076..fe5e6b8 100644 --- a/ASOFI3D/src/sofi3D.c +++ b/ASOFI3D/src/sofi3D.c @@ -31,11 +31,11 @@ #include "fd.h" #include "globvar.h" - +//#include "openacc.h" int main(int argc, char **argv) { //this needs to be moved to the json - ask Mahesh - int RTM_FLAG = 0; + int RTM_FLAG = 1; int ns, nt, nseismograms = 0, nf1, nf2; int lsnap, nsnap = 0, lsamp = 0, nlsamp = 0, buffsize; int ntr = 0, ntr_loc = 0, ntr_glob = 0, nsrc = 0, nsrc_loc = 0, ishot, nshots; /* removed variable "h", not in use*/ @@ -113,6 +113,11 @@ int main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD, &NP); MPI_Comm_rank(MPI_COMM_WORLD, &MYID); + + // Assign each MPI process to a GPU + // acc_init(acc_device_nvidia); + // acc_set_device_num(MYID, acc_device_nvidia); + setvbuf(stdout, NULL, _IONBF, 0); /* initialize clock for estimating runtime of program */ @@ -777,13 +782,9 @@ int main(int argc, char **argv) // Madagascar - if (RSF) madinput(RSFDEN,rho); - // mad_elastic(rho, pi, u, C11, C12, C13, C22, C23, C33, C44, C55, C66, taus, taup, eta); - - - - + if (RSF) madinput(RSFDEN,rho); + //mad_elastic(rho, pi, u, C11, C12, C13, C22, C23, C33, C44, C55, C66, taus, taup, eta); if (RUN_MULTIPLE_SHOTS) nshots = nsrc; @@ -980,6 +981,48 @@ int main(int argc, char **argv) lsamp = NDTSHIFT + 1; nlsamp = 1; + + + +//#pragma acc data copyin(vx[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vy[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vz[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1]) + +/* + + +vxyyx, vyzzy, vxzzx, vxxyyzz, vyyzz, vxxzz, vxxyy, +vxyyx_2, vyzzy_2, vxzzx_2, vxxyyzz_2, vyyzz_2, vxxzz_2, vxxyy_2, vxyyx_3, vyzzy_3, vxzzx_3, vxxyyzz_3, vyyzz_3, vxxzz_3, vxxyy_3, vxyyx_4, vyzzy_4, vxzzx_4, vxxyyzz_4, vyyzz_4, vxxzz_4, vxxyy_4 + +in: +out: sxx, syy, szz, sxy, syz, sxz,*/ + +//#pragma acc data copyin (C11,C12,C13,C33,C22,C23,C66ipjp,C44jpkp,C55ipkp) +//#pragma acc data copyout(sxy,syz,sxz,sxx,syy,szz) +/* +#pragma acc data copyin(xb[0],xb[1],yb[0],yb[1],zb[0],zb[1],nt) +#pragma acc data copyin(vx[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(vy[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(vz[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C11[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C12[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C13[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C22[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C23[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C33[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C66ipjp[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C44jpkp[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyin(C55ipkp[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +*/ +/* +#pragma acc data copyout(sxx[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyout(syy[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyout(szz[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyout(sxy[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyout(syz[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +#pragma acc data copyout(sxz[yb[0]-1:yb[1]+1][xb[0]-1:xb[1]+1][zb[0]-1:zb[1]+1]) +*/ + + + for (nt = 1; nt <= NT; nt++) { time_v_update[nt] = 0.0; diff --git a/ASOFI3D/src/sofi3D_1p_2017-02-06_12-42.map b/ASOFI3D/src/sofi3D_1p_2017-02-06_12-42.map new file mode 100644 index 0000000..427ecd7 Binary files /dev/null and b/ASOFI3D/src/sofi3D_1p_2017-02-06_12-42.map differ diff --git a/ASOFI3D/src/sofi3D_4p_2017-02-06_14-04.map b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-04.map new file mode 100644 index 0000000..216a5c1 Binary files /dev/null and b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-04.map differ diff --git a/ASOFI3D/src/sofi3D_4p_2017-02-06_14-12.map b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-12.map new file mode 100644 index 0000000..27ffaf3 Binary files /dev/null and b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-12.map differ diff --git a/ASOFI3D/src/sofi3D_4p_2017-02-06_14-15.map b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-15.map new file mode 100644 index 0000000..e1bba65 Binary files /dev/null and b/ASOFI3D/src/sofi3D_4p_2017-02-06_14-15.map differ diff --git a/ASOFI3D/src/sofi3D_8p_2017-02-06_14-20.map b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-20.map new file mode 100644 index 0000000..5774e97 Binary files /dev/null and b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-20.map differ diff --git a/ASOFI3D/src/sofi3D_8p_2017-02-06_14-22.map b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-22.map new file mode 100644 index 0000000..27c985b Binary files /dev/null and b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-22.map differ diff --git a/ASOFI3D/src/sofi3D_8p_2017-02-06_14-30.map b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-30.map new file mode 100644 index 0000000..1602e60 Binary files /dev/null and b/ASOFI3D/src/sofi3D_8p_2017-02-06_14-30.map differ diff --git a/ASOFI3D/src/update_s_ssg_CPML_elastic.c b/ASOFI3D/src/update_s_ssg_CPML_elastic.c index b4321fc..4ed9d14 100644 --- a/ASOFI3D/src/update_s_ssg_CPML_elastic.c +++ b/ASOFI3D/src/update_s_ssg_CPML_elastic.c @@ -95,8 +95,12 @@ double update_s_CPML_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz if (POS[1]==0){ +#pragma acc parallel +#pragma acc loop independent gang collapse(2) for (j=1;j<=NY;j++){ +#pragma acc loop independent for (i=1;i<=FW;i++){ +#pragma acc loop independent vector for (k=1;k<=NZ;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; @@ -193,8 +197,12 @@ double update_s_CPML_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz } if(POS[1]==NPROCX-1){ +#pragma acc parallel +#pragma acc loop independent for (j=1;j<=NY;j++){ +#pragma acc loop independent for (i=nx2+1;i<=nx2+FW;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; @@ -298,8 +306,12 @@ double update_s_CPML_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz } if((POS[2]==0 && FREE_SURF==0)){ +#pragma acc parallel +#pragma acc loop independent for (j=1;j<=FW;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; @@ -376,8 +388,12 @@ double update_s_CPML_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz } if(POS[2]==NPROCY-1){ +#pragma acc parallel +#pragma acc loop independent for (j=ny2+1;j<=ny2+FW;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; @@ -458,8 +474,12 @@ double update_s_CPML_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz if(POS[3]==0){ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=FW;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; @@ -566,8 +586,12 @@ maybe right here, maybe even inside this file... if(POS[3]==NPROCZ-1){ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz2+1;k<=nz2+FW;k++){ vxx = (b1*(vx[j][i][k]-vx[j][i-1][k])+b2*(vx[j][i+1][k]-vx[j][i-2][k]))/DX; diff --git a/ASOFI3D/src/update_s_ssg_elastic.c b/ASOFI3D/src/update_s_ssg_elastic.c index 14c0ea8..173da69 100644 --- a/ASOFI3D/src/update_s_ssg_elastic.c +++ b/ASOFI3D/src/update_s_ssg_elastic.c @@ -81,11 +81,18 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i case 2 : +//#pragma acc data copyin(vx[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vy[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vz[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1]) +//#pragma acc data copyin (C11,C12,C13,C33,C22,C23,C66ipjp,C44jpkp,C55ipkp) +//#pragma acc data copyout(sxy,syz,sxz,sxx,syy,szz) +#pragma acc parallel +#pragma acc loop independent for (j=ny1; j<=ny2; j++) { +#pragma acc loop independent for (i=nx1; i<=nx2; i++) { +#pragma acc loop independent for (k=nz1; k<=nz2; k++) { @@ -106,8 +113,8 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i c66ipjp=C66ipjp[j][i][k]*DT; c44jpkp=C44jpkp[j][i][k]*DT; c55ipkp=C55ipkp[j][i][k]*DT; - g=pi[j][i][k]; - f=2.0*u[j][i][k]; + //g=pi[j][i][k]; + //f=2.0*u[j][i][k]; c11=C11[j][i][k]; c12=C12[j][i][k]; @@ -148,10 +155,15 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i b2=-0.046414; } /* Holberg coefficients E=0.1 %*/ +//#pragma acc data copyin(vx[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vy[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1],vz[ny1-1:ny2+1][nx1-1:nx2+1][nz1-1:nz2+1]) +#pragma acc parallel +#pragma acc loop independent collapse(3) for (j=ny1; j<=ny2; j++) { +//#pragma acc loop independent for (i=nx1; i<=nx2; i++) { +//#pragma acc loop independent for (k=nz1; k<=nz2; k++) { @@ -173,7 +185,7 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i c44jpkp=C44jpkp[j][i][k]*DT; c55ipkp=C55ipkp[j][i][k]*DT; g=pi[j][i][k]; - f=2.0*u[j][i][k]; + f=2.0f*u[j][i][k]; c11=C11[j][i][k]; c12=C12[j][i][k]; @@ -237,6 +249,7 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i } } } +//#pragma acc end parallel break; case 6 : @@ -251,10 +264,14 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i b3=0.0081781; } /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent for (j=ny1; j<=ny2; j++) { +#pragma acc loop independent for (i=nx1; i<=nx2; i++) { +#pragma acc loop independent for (k=nz1; k<=nz2; k++) { @@ -351,10 +368,14 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i b4=-0.0026274; } /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent for (j=ny1; j<=ny2; j++) { +#pragma acc loop independent for (k=nz1; k<=nz2; k++) { +#pragma acc loop independent for (i=nx1; i<=nx2; i++) { @@ -464,10 +485,14 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i b5=0.001191; } /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent for (j=ny1; j<=ny2; j++) { +#pragma acc loop independent for (i=nx1; i<=nx2; i++) { +#pragma acc loop independent for (k=nz1; k<=nz2; k++) { @@ -586,10 +611,14 @@ double update_s_elastic(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, i b6=-0.00066667; } +#pragma acc parallel +#pragma acc loop independent for (j=ny1; j<=ny2; j++) { +#pragma acc loop independent for (i=nx1; i<=nx2; i++) { +#pragma acc loop independent for (k=nz1; k<=nz2; k++) { diff --git a/ASOFI3D/src/update_v_ssg.c b/ASOFI3D/src/update_v_ssg.c index 2f10c62..89c66a8 100755 --- a/ASOFI3D/src/update_v_ssg.c +++ b/ASOFI3D/src/update_v_ssg.c @@ -77,9 +77,13 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, b1=1.0; /* Taylor coefficients*/ if(FDCOEFF==2){ b1=1.00100; } /* Holberg coefficients E=0.1 %*/ - + +#pragma acc parallel +#pragma acc loop independent collapse(3) for (j=ny1;j<=ny2;j++){ +//#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +//#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*b1*(sxx[j][i+1][k]-sxx[j][i][k]); @@ -119,8 +123,12 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, if(FDCOEFF==2){ b1=1.1382; b2=-0.046414;} /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent collapse(3) for (j=ny1;j<=ny2;j++){ +//#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +//#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -159,9 +167,13 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, b1=75.0/64.0; b2=-25.0/384.0; b3=3.0/640.0; /* Taylor coefficients*/ if(FDCOEFF==2){ b1=1.1965; b2=-0.078804; b3=0.0081781;} /* Holberg coefficients E=0.1 %*/ - + +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+ @@ -227,8 +239,12 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, if(FDCOEFF==2){ b1=1.2257; b2=-0.099537; b3=0.018063; b4=-0.0026274;} /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+ @@ -303,8 +319,12 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, if(FDCOEFF==2){ b1=1.2415; b2=-0.11231; b3=0.026191; b4=-0.0064682; b5=0.001191;} /* Holberg coefficients E=0.1 %*/ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+ @@ -392,8 +412,12 @@ double update_v(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, b1=1.2508; b2=-0.12034; b3=0.032131; b4=-0.010142; b5=0.0029857; b6=-0.00066667;} +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz1;k<=nz2;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+ diff --git a/ASOFI3D/src/update_v_ssg_CPML.c b/ASOFI3D/src/update_v_ssg_CPML.c index 4f0be9c..8258cae 100644 --- a/ASOFI3D/src/update_v_ssg_CPML.c +++ b/ASOFI3D/src/update_v_ssg_CPML.c @@ -102,8 +102,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, /* boundaries in x-direction */ if (POS[1]==0){ +#pragma acc parallel +#pragma acc loop independent for (j=1;j<=NY;j++){ +#pragma acc loop independent for (i=1;i<=FW;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -174,8 +178,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, } if(POS[1]==NPROCX-1){ +#pragma acc parallel +#pragma acc loop independent for (j=1;j<=NY;j++){ +#pragma acc loop independent for (i=nx2+1;i<=nx2+FW;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -247,8 +255,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, } if((POS[2]==0 && FREE_SURF==0)){ +#pragma acc parallel +#pragma acc loop independent for (j=1;j<=FW;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -299,8 +311,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, } if(POS[2]==NPROCY-1){ +#pragma acc parallel +#pragma acc loop independent for (j=ny2+1;j<=ny2+FW;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=NZ;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -352,8 +368,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, /* boundaries in z-direction */ if(POS[3]==0){ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=1;k<=FW;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); @@ -384,8 +404,12 @@ double update_v_CPML(int nx1, int nx2, int ny1, int ny2, int nz1, int nz2, } if(POS[3]==NPROCZ-1){ +#pragma acc parallel +#pragma acc loop independent for (j=ny1;j<=ny2;j++){ +#pragma acc loop independent for (i=nx1;i<=nx2;i++){ +#pragma acc loop independent for (k=nz2+1;k<=nz2+FW;k++){ sxx_x = dx*(b1*(sxx[j][i+1][k]-sxx[j][i][k])+b2*(sxx[j][i+2][k]-sxx[j][i-1][k])); sxy_x = dx*(b1*(sxy[j][i][k]-sxy[j][i-1][k])+b2*(sxy[j][i+1][k]-sxy[j][i-2][k])); diff --git a/mpiInit.sh b/mpiInit.sh index 55fa545..6b20534 100755 --- a/mpiInit.sh +++ b/mpiInit.sh @@ -9,6 +9,8 @@ module load gcc module load mpi-openmpi module unload intel/16 + # default icc #module load intel + diff --git a/run_ASOFI3D.sh b/run_ASOFI3D.sh index 0344372..fcef190 100755 --- a/run_ASOFI3D.sh +++ b/run_ASOFI3D.sh @@ -19,7 +19,7 @@ cd - # Run the code echo "Run code" -./startSOFI3D.sh 8 +./startSOFI3D.sh $1 echo "OK" # Merge snapshots into visualizable ones