diff --git a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job index adc672ed6..3c6bd4feb 100644 --- a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job +++ b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job @@ -2,8 +2,8 @@ #SBATCH --partition=main #SBATCH -o log/VBT-kir-scan-%j-output.txt #SBATCH -e log/VBT-kir-scan-%j-error.txt -#SBATCH -t 0:59:00 -#SBATCH --time-min=0:59:00 +#SBATCH -t 3:59:00 +#SBATCH --time-min=3:59:00 #SBATCH -J VBT-kir-scan #SBATCH -A naiss2024-5-306 #SBATCH --nodes=1 @@ -81,7 +81,10 @@ unset DISPLAY export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom) -srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH --ipython_profile $IPYTHON_PROFILE &> log/setup_network_log.txt +# srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH --ipython_profile $IPYTHON_PROFILE &> log/setup_network_log.txt + +# Creating network in serial +srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH &> log/setup_network_log.txt # Quit ipyparallel diff --git a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only new file mode 100644 index 000000000..a26c578b7 --- /dev/null +++ b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only @@ -0,0 +1,93 @@ +#!/bin/bash -l +#SBATCH --partition=main +#SBATCH -o log/VBT-kir-scan-%j-output.txt +#SBATCH -e log/VBT-kir-scan-%j-error.txt +#SBATCH -t 3:59:00 +#SBATCH --time-min=3:59:00 +#SBATCH -J VBT-kir-scan +#SBATCH -A naiss2024-5-306 +#SBATCH --nodes=1 +#SBATCH --tasks-per-node=128 +#SBATCH --mem-per-cpu=930M +#SBATCH --mail-type=ALL + +NETWORK_PATH=networks/VBT_kir_scan +SIM_TIME=10 + +# This is used for NEURON +export N_WORKERS=$SLURM_NTASKS + +# This is used for ipyparallel and Snudda +export IPNWORKERS=50 + +# Clear old ipyparallel +export IPYTHONDIR="/cfs/klemming/scratch/${USER:0:1}/$USER/.ipython-${SLURM_JOB_ID}" +rm -r $IPYTHONDIR +export IPYTHON_PROFILE=default + +module load snic-env +source $HOME/Snudda/snudda_env/bin/activate +SNUDDA_DIR=/cfs/klemming/home/"${USER:0:1}"/$USER/Snudda + + +# If the BasalGangliaData directory exists, then use that for our data +if [[ -d "/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" ]]; then + export SNUDDA_DATA="/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" + echo "Setting SNUDDA_DATA to $SNUDDA_DATA" + rm mechanisms + ln -s $SNUDDA_DATA/neurons/mechanisms/ mechanisms +else + echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA" + rm mechanisms + ln -s ../../../snudda/data/neurons/mechanisms/ +fi + + +echo "Network path: "$NETWORK_PATH + +export PATH=$SNUDDA_DIR/snudda_env/bin/:$PATH +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH +export PYTHONPATH=$SNUDDA_DIR/snudda_env/lib/python3.9/ + +# This will stop NEURON from failing with "can't open DISPLAY" +unset DISPLAY + +export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom) + +echo "Skipping network creation" + + +############## + +rm -r x86_64 + +export CXX=CC +export CC=cc +export FC=ftn +export MPICC=cc +export MPICXX=CC + +CC --version + +echo "About to run nrnivmodl" +which nrnivmodl + +# Do we need this magic? +export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom) + +srun -n 1 nrnivmodl -incflags "-lltdl=/usr/lib64/libltdl.so.7 -lreadline=/lib64/libreadline.so.7 -lncurses=/lib64/libncurses.so.6.1" -loadflags "-DLTDL_LIBRARY=/usr/lib64/libltdl.so.7 -DREADLINE_LIBRARY=/lib64/libreadline.so.7 -DNCURSES_LIBRARY=/lib64/libncurses.so.6.1" mechanisms/ + + +for kir_factor in $(seq 1.0 -0.2 0.0); do + echo "Simulating kir_factor $kir_factor" + + # Do we need this magic? + export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom) + + srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/VBT_kir_scan/x86_64/special -mpi -python simulate_kir_scan.py $NETWORK_PATH --kir_factor $kir_factor --time $SIM_TIME --output $NETWORK_PATH/simulations/output_kir_$kir_factor.hdf5 +done + + +# Cleanup IPYTHONDIR +rm -r $IPYTHONDIR +