Migrate unit tests to GitHub (#175)

## Context
This PR migrates the unit test pipeline from GitLab to GitHub actions.
Based on the existing `simulation_test.yml`, it adds the model building
for `neocortex` and `neocortex-mutiscale` models, and launch the unit
tests via `pytest`. For testing data, sub circuits of `v5_sonata`,
`v5_sonata + gap junctions`, `sscx-v7-plasticity` are extracted by
`brainbuilder` from the original circuits in `gpfs` and are stored
locally.

## Scope
- [x]  Download neocortex mods from github repo
- [x] test-unit
- [x] test-integration
- [x] test-integration-e2e
- [x] test-scientific
- [x] test-scientific-ngv

Doc builds are tested on ReadTheDocs side for all PRs.

## Testing
Currently the follow tests are skipped:

## Review
* [x] PR description is complete
* [x] Coding style (imports, function length, New functions, classes or
files) are good
* [x] Unit/Scientific test added
* [x] Updated Readme, in-code, developer documentation

---------

Co-authored-by: Jorge Blanco Alonso <jorge.blancoalonso@epfl.ch>

.github/workflows/lint_check.yml

@@ -0,0 +1,19 @@
+name: Lint Check
+
+on: [pull_request, push, workflow_dispatch]
+
+jobs:
+  lint:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+
+    - name: Install flake8
+      run: pip install flake8-pyproject
+
+    - name: Run flake8
+      run: flake8 neurodamus tests

.github/workflows/simulation_test.yml

@@ -20,12 +20,13 @@ on:
       LIBSONATA_BRANCH:
         description: 'libsonata branch to use'
         required: false
+      NEURODAMUS_MODELS_BRANCH:
+        description: 'neurodamus-models branch to use'
+        required: false
 
 env:
-  NEURON_COMMIT_ID: '13654b3'
-  NEURON_BRANCH: ${{ inputs.NEURON_BRANCH || 'master' }}
-  LIBSONATA_REPORT_BRANCH: ${{ inputs.LIBSONATA_REPORT_BRANCH || '1.2.3' }}
-  LIBSONATA_BRANCH: ${{ inputs.LIBSONATA_BRANCH || 'v0.1.27' }}
+  NEURON_COMMIT_ID: 'c48d7d5'
+  RDMAV_FORK_SAFE: '1'
 
 jobs:
   simulation:
@@ -55,6 +56,16 @@ jobs:
         restore-keys: |
           ${{ matrix.os }}-pip-
 
+    - name: Get latest tags
+      run: |
+        echo "NEURON_BRANCH=${{ inputs.NEURON_BRANCH || github.event.inputs.NEURON_BRANCH || 'master' }}" >> $GITHUB_ENV
+        LIBSONATA_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonata/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "LIBSONATA_BRANCH=${{ inputs.LIBSONATA_BRANCH || github.event.inputs.LIBSONATA_BRANCH || '$LIBSONATA_LATEST' }}" >> $GITHUB_ENV
+        LIBSONATA_REPORT_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonatareport/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "LIBSONATA_REPORT_BRANCH=${{ inputs.LIBSONATA_REPORT_BRANCH || github.event.inputs.LIBSONATA_REPORT_BRANCH || '$LIBSONATA_REPORT_LATEST' }}" >> $GITHUB_ENV
+        NEURODAMUS_MODELS_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/neurodamus-models/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
+        echo "NEURODAMUS_MODELS_BRANCH=${{ inputs.NEURODAMUS_MODELS_BRANCH || github.event.inputs.NEURODAMUS_MODELS_BRANCH || '$NEURODAMUS_MODELS_LATEST' }}" >> $GITHUB_ENV
+
     - name: Get HEAD commit message and look for branches
       run: |
         COMMIT_MESSAGE=$(git log -1 --pretty=%B)
@@ -88,11 +99,13 @@ jobs:
         python -m pip install --upgrade pip setuptools
         pip install cython numpy wheel pkgconfig
 
+    - name: Add virtual environment to PATH
+      run: echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH
+
     - name: Install libsonata
       if: steps.cache-venv.outputs.cache-hit != 'true'
       run: |
-        . ./venv/bin/activate
-        CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata
+        CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata@${{ env.LIBSONATA_BRANCH }}
 
     - name: Cache libsonatareport
       id: cache-libsonatareport
@@ -126,7 +139,6 @@ jobs:
       if: steps.cache-neuron.outputs.cache-hit != 'true'
       run: |
         export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
-        sudo apt-get install flex libfl-dev bison ninja-build
         if [[ ${{ env.NEURON_BRANCH }} == 'master' ]]; then
           git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git
           cd nrn
@@ -135,7 +147,6 @@ jobs:
         else
           git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git --depth=1
         fi
-        . ./venv/bin/activate
         python -m pip install --upgrade pip -r nrn/nrn_requirements.txt
         cmake -B nrn/build -S nrn -G Ninja \
           -DPYTHON_EXECUTABLE=$(which python) \
@@ -153,44 +164,66 @@ jobs:
     - name: Install mpi4py
       if: steps.cache-venv.outputs.cache-hit != 'true'
       run: |
-        . ./venv/bin/activate
         pip install mpi4py
 
     - name: Build h5py with the local hdf5
       run: |
-        . ./venv/bin/activate
-        pip install cython numpy wheel pkgconfig
         CC="mpicc" HDF5_MPI="ON" HDF5_INCLUDEDIR=/usr/include/hdf5/mpich HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich \
           pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation
 
     - name: Install neurodamus
       run: |
-        . ./venv/bin/activate
-        pip install .
+        pip install .[full]
 
-    - name: Build models
+    - name: Build neocortex models
       run: |
         export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
         export PATH=$(pwd)/nrn/build/install/bin:$PATH
-        # copy mod files from the Zenodo link
-        wget --output-document="O1_mods.xz" --quiet "https://zenodo.org/record/8026353/files/O1_mods.xz?download=1"
-        tar -xf O1_mods.xz
-        cp -r mod tests/share/
-        cp neurodamus/data/mod/*.mod tests/share/mod/
-        ./docker/build_neurodamus.sh tests/share/mod
+        # Clone neurodamus-models repository
+        git clone --branch=${{ env.NEURODAMUS_MODELS_BRANCH }} https://github.com/BlueBrain/neurodamus-models.git
+        # Build neocortex model
+        DATADIR=$(python -c "import neurodamus; from pathlib import Path; print(Path(neurodamus.__file__).parent / 'data')")
+        cmake -B neurodamus-models/build -S neurodamus-models/ \
+          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install \
+          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
+          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
+          -DNEURODAMUS_CORE_DIR=${DATADIR} \
+          -DNEURODAMUS_MECHANISMS=neocortex \
+          -DNEURODAMUS_NCX_V5=ON
+        cmake --build neurodamus-models/build
+        cmake --install neurodamus-models/build
+        echo "NEURODAMUS_NEOCORTEX_ROOT=$(pwd)/neurodamus-models/install" >> $GITHUB_ENV;
+
+        # Build neocortex-multiscale model
+        echo "build neurodamus-neocortex-multiscale model"
+        cmake -B neurodamus-models/build_multiscale -S neurodamus-models/ \
+          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install_multiscale \
+          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
+          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
+          -DNEURODAMUS_CORE_DIR=${DATADIR} \
+          -DNEURODAMUS_MECHANISMS=neocortex \
+          -DNEURODAMUS_NCX_METABOLISM=ON \
+          -DNEURODAMUS_NCX_NGV=ON \
+          -DNEURODAMUS_ENABLE_CORENEURON=OFF
+        cmake --build neurodamus-models/build_multiscale
+        cmake --install neurodamus-models/build_multiscale
+        echo "NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT=$(pwd)/neurodamus-models/install_multiscale" >> $GITHUB_ENV;
 
-    - name: Example run
+    - name: load neocortex model
       run: |
-        . ./venv/bin/activate
-        if [[ ${{ matrix.python-version }} == 3.9.* ]]; then
-          export RDMAV_FORK_SAFE=1
-        fi;
-        export PYTHONPATH=$(pwd)/nrn/build/lib/python:$PYTHONPATH
-        cp neurodamus/data/hoc/* tests/share/hoc/
-        export HOC_LIBRARY_PATH=$(pwd)/tests/share/hoc
+        export PYTHONPATH=$(pwd)/nrn/build/install/lib/python:$PYTHONPATH
+        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_ROOT/share/neurodamus_neocortex/hoc
         export NEURODAMUS_PYTHON=$(pwd)/neurodamus/data
-        export CORENEURONLIB=$(pwd)/x86_64/libcorenrnmech.so
-        export PATH=$(pwd)/x86_64:$PATH
+        export CORENEURONLIB=$NEURODAMUS_NEOCORTEX_ROOT/lib/libcorenrnmech.so
+        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_ROOT/lib/libnrnmech.so
+        export PATH=$NEURODAMUS_NEOCORTEX_ROOT/bin:$PATH
+
+        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV;
+        echo "HOC_LIBRARY_PATH=$HOC_LIBRARY_PATH" >> $GITHUB_ENV;
+        echo "CORENEURONLIB=$CORENEURONLIB" >> $GITHUB_ENV;
+        echo "NRNMECH_LIB_PATH=$NRNMECH_LIB_PATH" >> $GITHUB_ENV;
+        echo "NEURODAMUS_PYTHON=$NEURODAMUS_PYTHON" >> $GITHUB_ENV;
+
         which special
 
         # launch simulation with NEURON
@@ -201,6 +234,35 @@ jobs:
         mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata_coreneuron.json
         ls reporting_coreneuron/*.h5
 
-    # - name: live debug session, comment out
-    #   if: failure()
-    #   uses: mxschmitt/action-tmate@v3
+    - name: Install test dependencies
+      run: |
+        pip install -r tests/requirements.txt
+
+    - name: test-unit
+      run: |
+        pytest tests/unit
+
+    - name: test-integration
+      run: |
+        pytest -x --forked tests/integration
+
+    - name: test-integration-e2e
+      run: |
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/integration-e2e
+
+    - name: test-scientific
+      run: |
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific
+
+    - name: test-scientific-ngv
+      run: |
+        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/share/neurodamus_neocortex/hoc
+        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/lib/libnrnmech.so
+        unset CORENEURONLIB
+        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific-ngv
+
+    - name: live debug session, comment out
+      if: failure()
+      uses: mxschmitt/action-tmate@v3
+      with:
+       timeout-minutes: 60  # or any other value you need

.gitlab-ci.yml

@@ -1,13 +1,7 @@
 include:
   - project: nse/ci
     file:
-      - /ci/lib/tox-bb5.yml
-      - /ci/jobs/build-package.yml
-      - /ci/jobs/publish-package.yml
-      - /ci/jobs/docs.yml
-      - /ci/jobs/publish-docs.yml
-  - project: hpc/gitlab-upload-logs
-    file: enable-upload.yml
+      - /ci/lib/common.yml
   - project: cs/gitlabci-templates
     file: /build-image-using-buildah.yml
 
@@ -25,44 +19,6 @@ variables:
     description: 'Name of the blueconfigs branch to test against'
   GIT_DEPTH: 100
 
-test-unit:
-  extends: .tox-template
-  variables:
-    TOXENV: flake8, unit
-
-test-integration:
-  extends: .tox-template
-  variables:
-    TOXENV: integration
-
-test-integration-e2e:
-  extends: .tox-template
-  variables:
-    TOXENV: bbp-model
-    TOX_OPTIONS: "-- tests/integration-e2e"
-    # modules need to be loaded in this certain order to avoid mixing of libraries
-    EXTRA_MODULES:
-      unstable:neurodamus-neocortex
-
-test-scientific:
-  extends: .tox-template
-  variables:
-    TOXENV: bbp-model
-    TOX_OPTIONS: "-- tests/scientific"
-    # modules need to be loaded in this certain order to avoid mixing of libraries
-    EXTRA_MODULES:
-      unstable:intel-oneapi-compilers
-      unstable:py-bluepysnap
-      unstable:neurodamus-neocortex
-
-test-scientific-ngv:
-  extends: .tox-template
-  variables:
-    TOXENV: bbp-model
-    TOX_OPTIONS: "-- tests/scientific-ngv"
-    EXTRA_MODULES:
-      unstable:neurodamus-neocortex-multiscale
-
 set_alt_branches:
   script:
     - grep '^[[:alnum:]_]*_BRANCH=' <<< "$CI_COMMIT_MESSAGE" > branch_variables.env || touch branch_variables.env
@@ -231,32 +187,3 @@ upload-to-docker-hub:
     - if: $CI_COMMIT_TAG
     - if: $CI_PIPELINE_SOURCE == "web"
       when: manual
-
-
-# The following stages are overrides. Don't rename
-# Always add the manual option to create sdist/wheel or upload
-build-package:
-  rules:
-    - when: on_success
-
-publish-package:
-  rules:
-    - when: manual
-      allow_failure: true
-
-docs:
-  variables:
-    TOXENV: docs
-  artifacts:
-    paths:
-      - docs/_build
-
-publish-docs:
-  variables:
-    SYS_PACKAGES: openssh-clients
-    PIP_PACKAGES: docs_internal_upload
-  script:
-    - docs-internal-upload --docs-path docs/_build
-  rules:
-    - when: manual
-      allow_failure: true

neurodamus/core/configuration.py

@@ -1039,7 +1039,7 @@ def _coreneuron_direct_mode(config: _SimConfig, run_conf):
 
 def get_debug_cell_gid(cli_options):
     gid = cli_options.get("dump_cell_state") if cli_options else None
-    if gid:
+    if gid is not None:
         try:
             # Convert to integer and adjust for sonata mode (0-based to 1-based indexing)
             gid = int(gid) + 1

neurodamus/io/sonata_config.py

@@ -336,9 +336,10 @@ def parsedProjections(self):
                         projection["Source"]
                 if projection.get("Type") == ConnectionTypes.GlioVascular:
                     for node_file_info in self._circuit_networks["nodes"]:
-                        for _, pop_info in node_file_info["populations"].items():
+                        for pop_name, pop_info in node_file_info["populations"].items():
                             if pop_info.get("type") == "vasculature":
-                                projection["VasculaturePath"] = node_file_info["nodes_file"]
+                                projection["VasculaturePath"] =\
+                                    self.circuits.node_population_properties(pop_name).elements_path
 
                 proj_name = "{0}__{1.source}-{1.target}".format(edge_pop_name, edge_pop)
                 projections[proj_name] = projection