From 089d4fea4930965e3ec9c0e8a86b73e67aaf5475 Mon Sep 17 00:00:00 2001
From: Johannes Hjorth <hjorth@kth.se>
Date: Mon, 28 Oct 2024 09:48:57 +0100
Subject: [PATCH] Updated slurm scripts

---
 .../VBT_kir_scan/Dardel_run_VBT_kir_scan.job  |  9 +-
 .../Dardel_run_VBT_kir_scan.job-sim-only      | 93 +++++++++++++++++++
 2 files changed, 99 insertions(+), 3 deletions(-)
 create mode 100644 examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only

diff --git a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job
index adc672ed6..3c6bd4feb 100644
--- a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job
+++ b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job
@@ -2,8 +2,8 @@
 #SBATCH --partition=main
 #SBATCH -o log/VBT-kir-scan-%j-output.txt
 #SBATCH -e log/VBT-kir-scan-%j-error.txt
-#SBATCH -t 0:59:00
-#SBATCH --time-min=0:59:00
+#SBATCH -t 3:59:00
+#SBATCH --time-min=3:59:00
 #SBATCH -J VBT-kir-scan
 #SBATCH -A naiss2024-5-306
 #SBATCH --nodes=1
@@ -81,7 +81,10 @@ unset DISPLAY
 
 export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
 
-srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH --ipython_profile $IPYTHON_PROFILE &> log/setup_network_log.txt
+# srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH --ipython_profile $IPYTHON_PROFILE &> log/setup_network_log.txt
+
+# Creating network in serial
+srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH &> log/setup_network_log.txt
 
 
 # Quit ipyparallel
diff --git a/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only
new file mode 100644
index 000000000..a26c578b7
--- /dev/null
+++ b/examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only
@@ -0,0 +1,93 @@
+#!/bin/bash -l
+#SBATCH --partition=main
+#SBATCH -o log/VBT-kir-scan-%j-output.txt
+#SBATCH -e log/VBT-kir-scan-%j-error.txt
+#SBATCH -t 3:59:00
+#SBATCH --time-min=3:59:00
+#SBATCH -J VBT-kir-scan
+#SBATCH -A naiss2024-5-306
+#SBATCH --nodes=1
+#SBATCH --tasks-per-node=128
+#SBATCH --mem-per-cpu=930M
+#SBATCH --mail-type=ALL
+
+NETWORK_PATH=networks/VBT_kir_scan
+SIM_TIME=10
+
+# This is used for NEURON
+export N_WORKERS=$SLURM_NTASKS
+
+# This is used for ipyparallel and Snudda
+export IPNWORKERS=50
+
+# Clear old ipyparallel
+export IPYTHONDIR="/cfs/klemming/scratch/${USER:0:1}/$USER/.ipython-${SLURM_JOB_ID}"
+rm -r $IPYTHONDIR
+export IPYTHON_PROFILE=default
+
+module load snic-env
+source $HOME/Snudda/snudda_env/bin/activate
+SNUDDA_DIR=/cfs/klemming/home/"${USER:0:1}"/$USER/Snudda
+
+
+# If the BasalGangliaData directory exists, then use that for our data
+if [[ -d "/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" ]]; then
+    export SNUDDA_DATA="/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data"
+    echo "Setting SNUDDA_DATA to $SNUDDA_DATA"
+    rm mechanisms
+    ln -s $SNUDDA_DATA/neurons/mechanisms/ mechanisms
+else
+    echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA"
+    rm mechanisms
+    ln -s ../../../snudda/data/neurons/mechanisms/
+fi
+
+
+echo "Network path: "$NETWORK_PATH
+
+export PATH=$SNUDDA_DIR/snudda_env/bin/:$PATH
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
+export PYTHONPATH=$SNUDDA_DIR/snudda_env/lib/python3.9/
+
+# This will stop NEURON from failing with "can't open DISPLAY"
+unset DISPLAY
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+echo "Skipping network creation"
+
+
+##############
+
+rm -r x86_64
+
+export CXX=CC
+export CC=cc
+export FC=ftn
+export MPICC=cc
+export MPICXX=CC
+
+CC --version
+
+echo "About to run nrnivmodl"
+which nrnivmodl
+
+# Do we need this magic?
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+srun -n 1 nrnivmodl -incflags "-lltdl=/usr/lib64/libltdl.so.7 -lreadline=/lib64/libreadline.so.7 -lncurses=/lib64/libncurses.so.6.1" -loadflags "-DLTDL_LIBRARY=/usr/lib64/libltdl.so.7 -DREADLINE_LIBRARY=/lib64/libreadline.so.7 -DNCURSES_LIBRARY=/lib64/libncurses.so.6.1" mechanisms/
+
+
+for kir_factor in $(seq 1.0 -0.2 0.0); do
+    echo "Simulating kir_factor $kir_factor"
+
+    # Do we need this magic?
+    export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+    srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/VBT_kir_scan/x86_64/special -mpi -python simulate_kir_scan.py $NETWORK_PATH --kir_factor $kir_factor --time $SIM_TIME --output $NETWORK_PATH/simulations/output_kir_$kir_factor.hdf5
+done
+
+
+# Cleanup IPYTHONDIR
+rm -r $IPYTHONDIR
+