-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathinitialization.cpl
125 lines (111 loc) · 4.76 KB
/
initialization.cpl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
WRITE
<*
/************************************************************************************************
* *
* Determine data for Pencil Decomposition and Scatter in MPI Space *
* *
************************************************************************************************
Author: Dr. Mirco Meazzo */
#include "fft_support.h"
getcwd(cwd_, sizeof(cwd_));
strcat(output_cwd_,cwd_);
strcat(output_cwd_,"/results");
if(rank==0) {
printf("Running in: %s\n", cwd_);
printf("Output in : %s\n", output_cwd_);
}
if( chdir(output_cwd_) !=0 )
perror(".:Error changing folder in outstats routine:.");
int nx,ny,nz;
nx = nx_+1; ny = ny_+3; nz = nz_*2+1;
int nzd = nzd_;
int nxd = nxd_;
// Length of the array along directions
int i_length = nxd;
int j_length = ny;
int k_length = nzd;
int npfast,npslow;
npfast=1; npslow=size;
// Recap header
if (rank == 0) {
printf("\n========================================================================================\n"
"\tDNS with %dx%dx%d modes investigated on %d procs\t\n"
"========================================================================================\n\n",
nx,ny,nz,size);
}
// Sanity check
if (nx<size) {
perror("\n\n\nInvalid Grid decomposition\nAborting simulation...\n\n\n");
abort();
}
MPI_Barrier(MPI_COMM_WORLD);
/******************************************** Remap Variables *******************************************/
fftw_plan plan, transpose;
ptrdiff_t alloc_local, local_x, local_x_start,local_z, local_z_start;
ptrdiff_t block_x, block_z;
ptrdiff_t n[2]; n[0]=nxd; n[1]=nzd;
// Define block per proc
block_def(size, nxd, nzd, &block_x, &block_z);
alloc_local = fftw_mpi_local_size_many_transposed(2, &n, 2, /*First 2 is rank, second is for complex*/
block_x, block_z, MPI_COMM_WORLD,
&local_x, &local_x_start,
&local_z, &local_z_start);
printf("[%d] local_x=%d\tlocal_z=%d\nstart_x=%d\tstart_z=%d\n", rank, local_x, local_z, local_x_start, local_z_start);
IMPLEMENTARE IL CHECK SULLE local_x/z: Se=0 allora modifica il block_x/z
// partitioning in z-pencil
int modes_per_proc[size], i, skipped_x=0, skipped_z=0;
//cores_handler(nx,size,modes_per_proc);
for(i=0;i<rank;i++){
skipped_x = skipped_x +modes_per_proc[i];
}
int z_klo = 0; // K fast
int z_khi = nzd-1;
int z_ilo = 0+ skipped_x; // I slow
int z_ihi = z_ilo + modes_per_proc[rank]-1;
printf("[Z-PENCIL] (k,i,j order)\t"
"On rank %d the coordinates are: "
"(%d,%d) -> (%d,%d)\n", rank, z_ilo, z_klo, z_ihi, z_khi );
// partitioning in x-pencil
//cores_handler(nzd,size,modes_per_proc);
for(i=0;i<rank;i++){
skipped_z = skipped_z +modes_per_proc[i];
}
int x_klo = 0+ skipped_z; // K slow
int x_khi = x_klo + modes_per_proc[rank]-1;
int x_ilo = 0; // I fast
int x_ihi = nx-1;
printf("[X-PENCIL] (i,j,k order)\t"
"On rank %d the coordinates are: "
"(%d,%d) -> (%d,%d)\n", rank, x_ilo, x_klo, x_ihi, x_khi );
/******************************************* Size Variables ******************************************/
localdim_x_ = 2*nxd*(x_khi-x_klo+1);
localdim_z_ = 2*(z_ihi-z_ilo+1)*nzd;
ilo_ = z_ilo; jlo_ = 0; klo_ = 0;
ihi_ = z_ihi; jhi_ = ny; khi_ = nz-1;
x_ilo_ = x_ilo, x_ihi_ = x_ihi, x_jlo_ = 0, x_jhi_ = ny, x_klo_ = x_klo, x_khi_ = x_khi;
z_ilo_ = z_ilo, z_ihi_ = z_ihi, z_jlo_ = 0, z_jhi_ = ny, z_klo_ = z_klo, z_khi_ = z_khi; *>
!// Take into account the negative modes
klo = klo - nz; khi = khi - nz;
jlo = jlo - 1; jhi = jhi - 1;
!// Fill the local array VV & V
ARRAY(ilo..ihi, klo..khi,-1..ny+1) OF MOMFLUX VV
ARRAY(ilo..ihi, klo..khi,-1..ny+1) OF VELOCITY V
ARRAY(ilo..ihi, klo..khi,1..(ny-1),1..maxtimelevels) OF rhstype oldrhs
rhstype memrhs(ilo..ihi,klo..khi,1..(ny-1))
INLINE FUNCTION newrhs(INTEGER ix,iz,iy)=memrhs(ix,iz,(iy+1000)MOD(3))
ARRAY(klo..khi,1..ny-1,-2..2) OF REAL D2vmat=0, etamat=0
ARRAY(ilo..ihi,klo..khi) OF STRUCTURE(COMPLEX u,v,w,vy,eta) bc0=0, bcn=0
POINTER TO ARRAY(0..localdim_x) OF REAL U_t
POINTER TO ARRAY(0..localdim_x) OF REAL V_t
POINTER TO ARRAY(0..localdim_x) OF REAL W_t
POINTER TO ARRAY(0..localdim_x) OF REAL uu
POINTER TO ARRAY(0..localdim_x) OF REAL uv
POINTER TO ARRAY(0..localdim_x) OF REAL vv
POINTER TO ARRAY(0..localdim_x) OF REAL vw
POINTER TO ARRAY(0..localdim_x) OF REAL uw
POINTER TO ARRAY(0..localdim_x) OF REAL ww
POINTER TO ARRAY(0..localdim_x) OF REAL sendbuf
POINTER TO ARRAY(0..localdim_x) OF REAL recvbuf
POINTER TO ARRAY(0..localdim_z*ny) OF REAL eta_write
POINTER TO ARRAY(0..localdim_z*ny) OF REAL v_write
set_time