-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathinstall_thor.bsh
446 lines (362 loc) · 14.2 KB
/
install_thor.bsh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
#
# Sequência de comandos usada para instalar e configurar a workstation thor
#
# Sistema: fedora server 37
#
# user: sampa
# passwd: padrão do grupo
#
# Não execute cegamente, provavelmete vai precisar de adaptações
#
# quase tudo (mas não tudo) deve ser executado como root
sudo su
dnf update
dnf group list
dnf groupinstall "C Development Tools and Libraries"
# dnf install @base-x gnome-shell gnome-terminal nautilus
dnf install gcc-gfortran.x86_64 cmake automake git boost
# systemctl set-default graphical.target
dnf install python3-cpuinfo
# Cockpit
# Se não estiver já instalado (durante a instalção do sistema)
dnf install cockpit
systemctl enable --now cockpit.socket
firewall-cmd --add-service=cockpit --permanent
firewall-cmd --reload
# Depois de instalado, você pode acessar o cockpit remotamente. Não precisa mais estar no terminal da máquina.
# Browse https://localhost:9090/ or https://your_ip_address:9090/ from any web browser and login with your user credentials.
# Se tiver GPU:
# NVIDIA
dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora37/x86_64/cuda-fedora37.repo
dnf -y module install nvidia-driver:latest-dkms
dnf install cuda-toolkit.x86_64 nvidia-driver-cuda nvidia-driver-libs
reboot
# Se o processador for intel, instale o oneAPI e não este
# AOCL = MKL da AMD:
# https://www.amd.com/en/developer/aocl.html
dnf install ./aocl-linux-gcc-4.0-1.x86_64.rpm
# OpenMPI
# Você pode criar várias versões do MPI com compiladores diferentes, como os da intel
# Aqui é openmpi + gnu
wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.bz2
tar -xjvf openmpi-4.1.4.tar.bz2
dnf install hwloc.x86_64 hwloc-devel.x86_64
CFLAGS='-march=native' CXXFLAGS='-march=native' FCFLAGS='-march=native' ./configure --prefix=/opt/openmpi-4.1.4-gcc --with-pmix --enable-mpi-cxx --enable-shared --enable-static --with-cuda
make -j4
make install
# SLURM
# Inicialmente, sugiro copiar a configuração de outra máquina
# https://slurm.schedmd.com/configurator.html
dnf install slurm.x86_64 slurm-slurmctld.x86_64 slurm-slurmd.x86_64 slurm-slurmdbd.x86_64 mariadb-server
systemctl enable slurmctld.service
systemctl enable slurmdbd.service
systemctl enable slurmd.service
systemctl enable munge.service
mungekey -c -v
chown munge: /etc/munge/munge.key
systemctl enable --now mariadb
mysql_secure_installation
mysql -u root -p
create user 'slurm'@'localhost' identified by 'password';
grant all on slurm_acct_db.* TO 'slurm'@'localhost';
# add to /etc/slurm/slurm.conf
# SelectTypeParameters=CR_Core_Memory
# disk health monitoring
vi /etc/smartmontools/smartd.conf
# comment out line:
# DEVICESCAN
# add:
#/dev/sda -a -o on -S on -s (S/../.././02|L/../../6/03) -m root,alberto.trj@gmail.com,alvaro.torrez@usp.br,alesandrokirch@gmail.com -M exec /usr/libexec/smartmontools/smartdnotify
#/dev/sdb -a -o on -S on -s (S/../.././02|L/../../6/03) -m root,alberto.trj@gmail.com,alvaro.torrez@usp.br,alesandrokirch@gmail.com -M exec /usr/libexec/smartmontools/smartdnotify
# Monitoramento:
# Performance Co-Pilot + Grafana:
dnf install lm_sensors.x86_64 cockpit-pcp.x86_64 redis
sensors-detect
# enable cockpit-pcp on cockpit
# In cockpit:
# enable "Collect Metrics" in "Metrics Settings"
# enable "Export to Network" in "Metrics Settings"
dnf install grafana grafana-pcp pcp-gui pcp-pmda-slurm.x86_64 pcp-pmda-nvidia-gpu.x86_64 pcp-pmda-lmsensors.x86_64 pcp-pmda-summary pcp-pmda-smart RediSearch.x86_64
systemctl enable --now grafana-server
cd /var/lib/pcp/pmdas/nvidia
./Install
cd /var/lib/pcp/pmdas/lmsensors/
sensors -u
./Install
cd /var/lib/pcp/pmdas/slurm/
./Install
# O plugin do slurm para o pcp deu erro na hora de instalar
cd /var/lib/pcp/pmdas/smart/
./Install
cd ../summary/
./Install
systemctl enable redis pmlogger pmproxy pmlogger
systemctl start redis pmlogger pmproxy pmlogger
firewall-cmd --add-service=grafana --permanent
firewall-cmd --add-service=pmproxy --permanent
firewall-cmd --reload
# Página de monitoramento (grafana + pcp) pode ser acessada via
# http://thor.dyn.if.usp.br:3000
echo 'Monitoring tool: http://thor.dyn.if.usp.br:3000' > /etc/motd.d/PCP-grafana
# Módulos
# Provavelmente você vai precisar modificar esta parte de acordo com a máquina
# Fica como referência:
dnf install environment-modules
mkdir /opt/modulesfiles
cat << EOF > /etc/profile.d/cluster-modules.sh
export MODULEPATH=\${MODULEPATH}:/opt/AMD/aocl/aocl-linux-gcc-4.0:/opt/modulesfiles
EOF
mkdir -p /opt/modulesfiles/openmpi/gnu
cat << EOF > /opt/modulesfiles/openmpi/gnu/4.1.4
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "OpenMPI + GNU 12.2.1 64-bit Compiler"
# Set intermediate variables
set _flavor "gcc"
set __ompiroot "/opt/openmpi-4.1.4-\${_flavor}"
# Setup environment variables
prepend-path LD_LIBRARY_PATH \${__ompiroot}/lib
prepend-path LIBRARY_PATH \${__ompiroot}/lib
prepend-path CPATH \${__ompiroot}/include
prepend-path PKG_CONFIG_PATH \${__ompiroot}/lib/pkgconfig
# PATH
prepend-path PATH "\${__ompiroot}/bin"
# MANPATH
append-path MANPATH "\${__ompiroot}/share/man"
EOF
# VASP
# como usuário "sampa"
su - sampa
module load aocl-linux-gcc-4.0_module
module load openmpi/gnu/4.1.4
# compilar vasp
cd ~/Packages/vasp.6.3.0
cp arch/makefile.include.gnu_ompi_aocl ./makefile.include
# colocar:
# AMDBLIS_ROOT, AMDLIBFLAME_ROOT, AMDSCALAPACK_ROOT, AMDFFTW_ROOT =
# /opt/AMD/aocl/aocl-linux-gcc-4.0
make DEPS=1 -j4 all
for i in std gam ncl; do cp build/$i/vasp vasp_$i; done
# Pra compilar a versão 6.X do vasp pra gpu precisa do Nvidia HPC SDK
# Cheque a versão mais recente/compatível/necessária
dnf config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo
dnf install nvhpc-23.1
echo 'export MODULEPATH=${MODULEPATH}:/opt/AMD/aocl/aocl-linux-gcc-4.0:/opt/modulesfiles:/opt/nvidia/hpc_sdk/modulefiles' > /etc/profile.d/cluster-modules.sh
# Pra compilar vasp + gpu + aocl:
source /etc/profile.d/cluster-modules.sh
module purge
module load aocl-linux-gcc-4.0_module nvhpc-hpcx/23.1
cp arch/makefile.include.nvhpc_omp_acc ./makefile.include
vi ./makefile.include
# linhas alteradas:
# ajuste tp= ... conforme o processador e gpu=... conforme a gpu
# ajuste o resto conforme o resto
FC = mpif90 -acc -gpu=cc86,cuda12.0 -mp -tp=zen3
FCL = mpif90 -acc -gpu=cc86,cuda12.0 -mp -tp=zen3 -c++libs
FC_LIB = nvfortran -tp=zen3
CC_LIB = nvc -tp=zen3 -w
BLAS = -lblis
LAPACK = -lflame
FFTW_ROOT ?= /opt/AMD/aocl/aocl-linux-gcc-4.0
#
make DEPS=1 -j8 all
for i in std gam ncl; do cp build/$i/vasp vasp_gpu_aocl_$i; done
# como "sampa"
vi ./makefile.include
# linhas alteradas:
BLAS = -lblas
LAPACK = -llapack
make DEPS=1 -j8 all
for i in std gam ncl; do cp build/$i/vasp vasp_gpu_$i; done
# como root:
mkdir -p /opt/modulesfiles/vasp/6.3.0
cd /opt/modulesfiles/vasp/6.3.0
cat << EOF > cpu
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "VASP 6.3.0 comiled with OpenMPI + AOCL"
# Load required dependencies
if { ![ is-loaded openmpi/gnu/4.1.4 ] } { module load openmpi/gnu/4.1.4 }
if { ![ is-loaded aocl-linux-gcc-4.0_module ] } { module load aocl-linux-gcc-4.0_module }
# PATH
prepend-path PATH "/opt/bin"
EOF
cat << EOF > gpu
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "VASP 6.3.0 comiled with Nvidia HPC SDK"
# Load required dependencies
if { ![ is-loaded nvhpc-hpcx/23.1 ] } { module load nvhpc-hpcx/23.1 }
# PATH
prepend-path PATH "/opt/bin"
EOF
# como root:
mkdir /opt/bin
cp ./vasp_* /opt/bin
# SIESTA
# https://docs.siesta-project.org/projects/siesta/en/latest/how-to/build-install.html
# como "sampa":
cd /home/sampa/Packages/siesta/siesta-v4.1.5/Obj
module purge
module load openmpi/gnu/4.1.4 aocl-linux-gcc-4.0_module
vi arch.make
# editar:
# compiladores gcc, gfortran e openmpi
# blas, lapack da aocl: blis e flame
# otimização: -march=native -mtune=native
# para a atual versão do gfortran precisa usar: -fallow-argument-mismatch
# adicionar "-lalm -lm" em LIBS
make
# como root:
cp siesta /opt/bin/
mkdir -p /opt/modulesfiles/siesta
cd /opt/modulesfiles/siesta
cat << EOF > 4.1.5
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "SIESTA 4.1.5 comiled with OpenMPI + AOCL"
# Load required dependencies
if { ![ is-loaded openmpi/gnu/4.1.4 ] } { module load openmpi/gnu/4.1.4 }
if { ![ is-loaded aocl-linux-gcc-4.0_module ] } { module load aocl-linux-gcc-4.0_module }
# PATH
prepend-path PATH "/opt/bin"
EOF
# LAMMPS
dnf install python3-devel python3-pip dnf eigen3-devel
wget https://download.lammps.org/tars/lammps-stable.tar.gz
tar -xzvf lammps-stable.tar.gz
cd lammps-23Jun2022
mkdir build; cd build
module load aocl-linux-gcc-4.0_module openmpi/gnu/4.1.4 cuda/12.1
cat << EOF > ../cmake.bsh
cmake \
-C ../cmake/presets/most.cmake \
-D PKG_GPU=yes \
-D GPU_API=cuda \
-D GPU_ARCH=86 \
-D CUDA_MPS_SUPPORT=yes \
-D PKG_COLVARS=yes \
-D PKG_KSPACE=yes \
-D PKG_OPENMP=yes \
-D PKG_OPT=yes \
-D PKG_PYTHON=yes \
-D PKG_POEMS=yes \
-D PKG_REAXFF=yes \
-D FFT=FFTW3 \
-D FFT_FFTW_THREADS=off \
-D FFTW3_INCLUDE_DIR=/opt/AMD/aocl/aocl-linux-gcc-4.0/include \
-D FFTW3_LIBRARY=/opt/AMD/aocl/aocl-linux-gcc-4.0/lib/libfftw3.so \
-D USE_INTERNAL_LINALG=no \
-D LAPACK_LIBRARIES=-lflame \
-D CMAKE_TUNE_FLAGS=-march=native \
-D CMAKE_INSTALL_PREFIX=/opt/lammps-23Jun2022 \
../cmake
EOF
../cmake.bsh
cmake --build . -j8
# root:
module load aocl-linux-gcc-4.0_module openmpi/gnu/4.1.4
make install
mkdir -p /opt/modulesfiles/lammps/23Jun2022
cd /opt/modulesfiles/lammps/23Jun2022
cat << EOF > gpu
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "LAMMPS 23Jun2022 comiled with OpenMPI + GNU + CUDA"
# Load required dependencies
if { ![ is-loaded openmpi/gnu/4.1.4 ] } { module load openmpi/gnu/4.1.4 }
if { ![ is-loaded aocl-linux-gcc-4.0_module ] } { module load aocl-linux-gcc-4.0_module }
# Environment variables
prepend-path PATH "/opt/lammps-23Jun2022/bin"
prepend-path LIBRARY_PATH "/opt/lammps-23Jun2022/lib"
prepend-path MANPATH "/opt/lammps-23Jun2022/share/man"
EOF
# monitoramento de uso do /home
cat << EOF > /etc/cron.daily/check_home_usage.bsh
du -s \$(ls -1d /home/*/ \
| grep -v 'lost+found\|slurm') \
| awk -v FS='/' -v ds=\$(df | grep home | awk '{print \$2}') '
BEGIN{printf("\nUso da partição home:\n")}
{
printf("%12s %6.2f %\n", \$3, 100*\$1/ds)
}' \
> /etc/motd.d/home-disk-usage
EOF
chmod +x /etc/cron.daily/check_home_usage.bsh
/etc/cron.daily/check_home_usage.bsh
#
dnf install python3-numpy.x86_64 python2-numpy.x86_64
dnf install gnuplot
# apptainer
dnf install apptainer.x86_64
# KrelLBM (código do Surmas):
#
# BOOST
#
mkdir ~/lib
cd ~/lib
wget https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.bz2
tar -xjvf boost_1_81_0.tar.bz2
cd boost_1_81_0/
./bootstrap.sh --prefix=/opt/boost-1.81.0 --show-libraries
CFLAGS='-march=native' CXXFLAGS='-march=native' ./bootstrap.sh --prefix=/opt/boost-1.81.0
./b2 install --prefix=/opt/boost-1.81.0 variant=release optimization=speed cxxflags='-march=native' cflags='-march=native'
sudo ./b2 install --prefix=/opt/boost-1.81.0 variant=release optimization=speed cxxflags='-march=native' cflags='-march=native'
sudo mkdir -p /opt/modulesfiles/boost
sudo cat << EOF > /opt/modulesfiles/boost/1.81.0
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "Boost C++ libraries + GNU 12.2.1 64-bit Compiler"
set BOOST_ROOT "/opt/boost-1.81.0"
# Setup environment variables
prepend-path LD_LIBRARY_PATH \${BOOST_ROOT}/lib
prepend-path LIBRARY_PATH \${BOOST_ROOT}/lib
prepend-path CPATH \${BOOST_ROOT}/include
EOF
#
# METIS
#
cd ~/lib
wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
tar -xzvf metis-5.1.0.tar.gz
cd metis-5.1.0/
CFLAGS='-march=native' make config shared=1 prefix=/opt/metis-5.1.0
CFLAGS='-march=native' make
sudo make install
sudo mkdir -p /opt/modulesfiles/metis
sudo cat << EOF > /opt/modulesfiles/metis/5.1.0
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "Metis 5.1.0 + GNU 12.2.1 64-bit Compiler"
set METIS_ROOT "/opt/metis-5.1.0"
# Setup environment variables
prepend-path LD_LIBRARY_PATH \${METIS_ROOT}/lib
prepend-path LIBRARY_PATH \${METIS_ROOT}/lib
prepend-path CPATH \${METIS_ROOT}/include
prepend-path PATH \${METIS_ROOT}/bin
EOF
#
# PARMETIS
#
cd ~/lib
wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz
tar -xzvf parmetis-4.0.3.tar.gz
cd parmetis-4.0.3/
module load openmpi/gnu/4.1.4
CFLAGS='-march=native' make config shared=1 prefix=/opt/parmetis-4.0.3
CFLAGS='-march=native' make
sudo make install
sudo mkdir -p /opt/modulesfiles/parmetis
sudo cat << EOF > /opt/modulesfiles/parmetis/4.0.3
#%Module1.0###################################################################
# ###### Component Specific env vars setup ###################################
module-whatis "ParMetis 4.0.3 + OpenMPI + GNU 12.2.1 64-bit Compiler"
set PARMETIS_ROOT "/opt/parmetis-4.0.3"
# Load dependencies
if { ![ is-loaded openmpi/gnu/4.1.4 ] } { module load openmpi/gnu/4.1.4 }
if { ![ is-loaded metis/5.1.0 ] } { module load metis/5.1.0 }
# Setup environment variables
prepend-path LD_LIBRARY_PATH \${PARMETIS_ROOT}/lib
prepend-path LIBRARY_PATH \${PARMETIS_ROOT}/lib
prepend-path CPATH \${PARMETIS_ROOT}/include
prepend-path PATH \${PARMETIS_ROOT}/bin