Merge pull request #433 from Hjorthmedh/neuromod_redux2

VBT simulations KIR variation

examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job

@@ -0,0 +1,128 @@
+#!/bin/bash -l
+#SBATCH --partition=main
+#SBATCH -o log/VBT-kir-scan-%j-output.txt
+#SBATCH -e log/VBT-kir-scan-%j-error.txt
+#SBATCH -t 3:59:00
+#SBATCH --time-min=3:59:00
+#SBATCH -J VBT-kir-scan
+#SBATCH -A naiss2024-5-306
+#SBATCH --nodes=1
+#SBATCH --tasks-per-node=128
+#SBATCH --mem-per-cpu=930M
+#SBATCH --mail-type=ALL
+
+NETWORK_PATH=networks/VBT_kir_scan
+SIM_TIME=10
+
+# This is used for NEURON
+export N_WORKERS=$SLURM_NTASKS
+
+# This is used for ipyparallel and Snudda
+export IPNWORKERS=50
+
+# Clear old ipyparallel
+export IPYTHONDIR="/cfs/klemming/scratch/${USER:0:1}/$USER/.ipython-${SLURM_JOB_ID}"
+rm -r $IPYTHONDIR
+export IPYTHON_PROFILE=default
+
+module load snic-env
+source $HOME/Snudda/snudda_env/bin/activate
+SNUDDA_DIR=/cfs/klemming/home/"${USER:0:1}"/$USER/Snudda
+
+
+# If the BasalGangliaData directory exists, then use that for our data
+if [[ -d "/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" ]]; then
+    export SNUDDA_DATA="/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data"
+    echo "Setting SNUDDA_DATA to $SNUDDA_DATA"
+    rm mechanisms
+    ln -s $SNUDDA_DATA/neurons/mechanisms/ mechanisms
+else
+    echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA"
+    rm mechanisms
+    ln -s ../../../snudda/data/neurons/mechanisms/
+fi
+
+
+# Start ipyparallel
+#.. Start the ipcontroller
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+srun -n 1 -N 1 -c 2 --exact --overlap --mem=0 ./../ipcontroller_new.sh &
+
+
+echo ">>> waiting 60s for controller to start"
+sleep 60 
+
+#.. Read in CONTROLLERIP
+CONTROLLERIP=$(<controller_ip.txt)
+
+##.. Start the engines
+echo ">>> starting ${IPNWORKERS} engines "
+echo "SLURM_JOB_NUM_NODES={SLURM_JOB_NUM_NODES}"
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+srun -n ${IPNWORKERS} -c 2 -N ${SLURM_JOB_NUM_NODES} --exact --overlap --mem=0 ipengine \
+--location=${CONTROLLERIP} --profile=${IPYTHON_PROFILE} --mpi \
+--ipython-dir=${IPYTHONDIR}  --timeout=30.0 c.EngineFactory.max_heartbeat_misses=10  c.MPI.use='mpi4py' \
+1> ipe_${SLURM_JOBID}.out 2> ipe_${SLURM_JOBID}.err &
+
+
+echo ">>> waiting 60s for engines to start"
+sleep 30
+
+
+echo "Network path: "$NETWORK_PATH
+
+export PATH=$SNUDDA_DIR/snudda_env/bin/:$PATH
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
+export PYTHONPATH=$SNUDDA_DIR/snudda_env/lib/python3.9/
+
+# This will stop NEURON from failing with "can't open DISPLAY"
+unset DISPLAY
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+# srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH --ipython_profile $IPYTHON_PROFILE &> log/setup_network_log.txt
+
+# Creating network in serial
+srun -n 1 -N 1 --exact --overlap --mem=0 python setup_kir_scan.py $NETWORK_PATH &> log/setup_network_log.txt
+
+
+# Quit ipyparallel
+
+python ../ipcontroller_shutdown.py
+
+
+##############
+
+rm -r x86_64
+
+export CXX=CC
+export CC=cc
+export FC=ftn
+export MPICC=cc
+export MPICXX=CC
+
+CC --version
+
+echo "About to run nrnivmodl"
+which nrnivmodl
+
+# Do we need this magic?
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+srun -n 1 nrnivmodl -incflags "-lltdl=/usr/lib64/libltdl.so.7 -lreadline=/lib64/libreadline.so.7 -lncurses=/lib64/libncurses.so.6.1" -loadflags "-DLTDL_LIBRARY=/usr/lib64/libltdl.so.7 -DREADLINE_LIBRARY=/lib64/libreadline.so.7 -DNCURSES_LIBRARY=/lib64/libncurses.so.6.1" mechanisms/
+
+
+for kir_factor in $(seq 1.0 -0.2 0.0); do
+    echo "Simulating kir_factor $kir_factor"
+
+    # Do we need this magic?
+    export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+    srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/VBT_kir_scan/x86_64/special -mpi -python simulate_kir_scan.py $NETWORK_PATH --kir_factor $kir_factor --time $SIM_TIME --output $NETWORK_PATH/simulations/output_kir_$kir_factor.hdf5
+done
+
+
+# Cleanup IPYTHONDIR
+rm -r $IPYTHONDIR
+

examples/parallel/KTH_PDC/VBT_kir_scan/Dardel_run_VBT_kir_scan.job-sim-only

@@ -0,0 +1,93 @@
+#!/bin/bash -l
+#SBATCH --partition=main
+#SBATCH -o log/VBT-kir-scan-%j-output.txt
+#SBATCH -e log/VBT-kir-scan-%j-error.txt
+#SBATCH -t 8:59:00
+#SBATCH --time-min=8:59:00
+#SBATCH -J VBT-kir-scan
+#SBATCH -A naiss2024-5-306
+#SBATCH --nodes=10-20
+#SBATCH --tasks-per-node=128
+#SBATCH --mem-per-cpu=930M
+#SBATCH --mail-type=ALL
+
+NETWORK_PATH=networks/VBT_kir_scan
+SIM_TIME=10
+
+# This is used for NEURON
+export N_WORKERS=$SLURM_NTASKS
+
+# This is used for ipyparallel and Snudda
+export IPNWORKERS=50
+
+# Clear old ipyparallel
+export IPYTHONDIR="/cfs/klemming/scratch/${USER:0:1}/$USER/.ipython-${SLURM_JOB_ID}"
+rm -r $IPYTHONDIR
+export IPYTHON_PROFILE=default
+
+module load snic-env
+source $HOME/Snudda/snudda_env/bin/activate
+SNUDDA_DIR=/cfs/klemming/home/"${USER:0:1}"/$USER/Snudda
+
+
+# If the BasalGangliaData directory exists, then use that for our data
+if [[ -d "/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" ]]; then
+    export SNUDDA_DATA="/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data"
+    echo "Setting SNUDDA_DATA to $SNUDDA_DATA"
+    rm mechanisms
+    ln -s $SNUDDA_DATA/neurons/mechanisms/ mechanisms
+else
+    echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA"
+    rm mechanisms
+    ln -s ../../../snudda/data/neurons/mechanisms/
+fi
+
+
+echo "Network path: "$NETWORK_PATH
+
+export PATH=$SNUDDA_DIR/snudda_env/bin/:$PATH
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
+export PYTHONPATH=$SNUDDA_DIR/snudda_env/lib/python3.9/
+
+# This will stop NEURON from failing with "can't open DISPLAY"
+unset DISPLAY
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+echo "Skipping network creation"
+
+
+##############
+
+rm -r x86_64
+
+export CXX=CC
+export CC=cc
+export FC=ftn
+export MPICC=cc
+export MPICXX=CC
+
+CC --version
+
+echo "About to run nrnivmodl"
+which nrnivmodl
+
+# Do we need this magic?
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+srun -n 1 nrnivmodl -incflags "-lltdl=/usr/lib64/libltdl.so.7 -lreadline=/lib64/libreadline.so.7 -lncurses=/lib64/libncurses.so.6.1" -loadflags "-DLTDL_LIBRARY=/usr/lib64/libltdl.so.7 -DREADLINE_LIBRARY=/lib64/libreadline.so.7 -DNCURSES_LIBRARY=/lib64/libncurses.so.6.1" mechanisms/
+
+
+for kir_factor in $(seq 1.0 -0.2 0.0); do
+    echo "Simulating kir_factor $kir_factor"
+
+    # Do we need this magic?
+    export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+
+    srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/VBT_kir_scan/x86_64/special -mpi -python simulate_kir_scan.py $NETWORK_PATH --kir_factor $kir_factor --time $SIM_TIME --output $NETWORK_PATH/simulations/output_kir_$kir_factor.hdf5
+done
+
+
+# Cleanup IPYTHONDIR
+rm -r $IPYTHONDIR
+

examples/parallel/KTH_PDC/VBT_kir_scan/README.md

@@ -0,0 +1,39 @@
+# Running detailed simulations of Striatum for VBT
+
+This code creates a striatal network with 10000 neurons based on touch detection using Snudda. The network is simulated, first in the WT configuration, then with decreased KIR current in dSPN and iSPN neurons.
+
+## Snudda installation instructions
+
+[Installation instructions](https://github.com/Hjorthmedh/Snudda/wiki/1.-User-installation)
+
+This code also requires [BasalGangliaData](https://github.com/hjorthmedh/BasalGangliaData)
+
+
+## Quick start:
+
+To build the network and run it (note this SLURM file only uses one node, so will take 11 hours per simulation on Dardel).
+
+```
+Dardel_run_VBT_kir_scan.job
+```
+
+and to run only the simulations with 10-20 nodes for shorter run time:
+
+```
+Dardel_run_VBT_kir_scan.job-sim-only
+```
+
+## Code structure
+
+The program ```setup_kir_scan.py``` takes ```network_path``` as an argument, and optionally ```--python_profile default``` and creates the network that is defined in the ```config/network.json``` file. It is a striatal network with 10000 neurons containing dSPN, iSPN, FS, LTS and ChIN. The input is specified in ```input.json```, here a steady 2Hz input to each synapse on the neurons. 
+
+The program ```simulate_kir_scan.py``` takes ```network_path```, ```--kir_factor``` (between 0.0 and 1.0, ie WT), ```--output``` specifying the simulation output file and ```--time``` the duration (in seconds) of the simulation.
+
+Both these programs are called from ```Dardel_run_VBT_kir_scan.job```. In the default configuration the kir_factor used is 1.0, 0.8, 0.6, 0.4, 0.2 and 0.
+
+The network structure can be inspected with ```snudda_load your_simulation_file.hdf5```, to get instructions use ```snudda_load --help```. This uses ```snudda/utils/load.py``` which you can also call directly in python.
+
+The simulation data can be loaded using ```snudda/utils/load_network_simulation.py```, if you want to plot individual traces you can use ```snudda_plot_traces```, use ```--help``` as an option to get instructions.
+
+
+

examples/parallel/KTH_PDC/VBT_kir_scan/config/network.json

@@ -1,18 +1,19 @@
 {
     "network_path": "networks/wt_kir_scan_1",
-    "snudda_data": "/home/hjorth/HBP/2024-10-03/BasalGangliaData/data/",
+    "snudda_data": "/cfs/klemming/home/h/hjorth/BasalGangliaData/data",
+    "!snudda_data": "/home/hjorth/HBP/2024-10-03/BasalGangliaData/data/",
 
     "random_seed": {
 	"master_seed": 1234	
     },
 
     "regions": {
 	"Striatum" : {
-	    "num_neurons": 10,
+	    "num_neurons": 10000,
 	    "volume": {
 		"mesh_file": "$SNUDDA_DATA/meshes/Striatum-dorsal-left-hemisphere.obj",
 		"d_min": 1.5e-5,
-		"num_putative_points": 100000,
+		"num_putative_points": 500000,
 		"random_seed": 123456
 	    },
 	    "neurons": ["neurons/dspn.json",

examples/parallel/KTH_PDC/VBT_kir_scan/setup_kir_scan.py

@@ -6,9 +6,17 @@
 
     parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
     parser.add_argument("network_path", type=str)
+    parser.add_argument("--ipython_profile", default=None)
     args = parser.parse_args()
 
-    snd = Snudda(network_path=args.network_path)
+    if args.ipython_profile:
+        parallel_flag=True
+    else:
+        parallel_flag=False
+
+    snd = Snudda(network_path=args.network_path, parallel=parallel_flag,
+                 ipython_profile=args.ipython_profile)
+
     snd.import_config("config/network.json", overwrite=True)
     snd.create_network(honor_morphology_stay_inside=False)
 

examples/parallel/KTH_PDC/VBT_kir_scan/simulate_kir_scan.py

@@ -9,17 +9,17 @@ def simulate(network_path, output_path, kir_factor=1.0, time=1.0):
     # Next we need to reduce kir by a factor
     print("Reduce the KIR channel conductance in SPN")
 
-    print(f"Before: {sim.neurons[0].icell.soma[0](0.5).kir_ms.gbar = }")
+    if 0 in sim.neurons and "SPN" in sim.neurons[0].name:
+        print(f"Before: {sim.neurons[0].icell.soma[0](0.5).kir_ms.gbar = }")
 
     for sec in sim.sim.neuron.h.allsec():
         for seg in sec:
             if sim.sim.neuron.h.ismembrane("kir_ms", sec=sec):
                 # Scale the conductance parameter for each segment
                 seg.kir_ms.gbar *= kir_factor
 
-
-    print(f"After: {sim.neurons[0].icell.soma[0](0.5).kir_ms.gbar = }")
-
+    if 0 in sim.neurons and "SPN" in sim.neurons[0].name:
+        print(f"After: {sim.neurons[0].icell.soma[0](0.5).kir_ms.gbar = }")
 
     sim.run(t=time*1e3)
     sim.write_output()