simulation_test.yml

name: neurodamus build and test
on:
  schedule:
    # every evening at 20:00 UTC
    - cron: '0 20 * * *'
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      NEURON_BRANCH:
        description: 'NEURON branch to use'
        required: false
      LIBSONATA_REPORT_BRANCH:
        description: 'libsonatareport branch to use'
        required: false
      LIBSONATA_BRANCH:
        description: 'libsonata branch to use'
        required: false
      NEURODAMUS_MODELS_BRANCH:
        description: 'neurodamus-models branch to use'
        required: false

env:
  NEURON_COMMIT_ID: 'c48d7d5'
  RDMAV_FORK_SAFE: '1'

jobs:
  simulation:
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ubuntu-22.04]
        python-version: ['3.9.20', '3.10.15', '3.11.10', '3.12.6']
    steps:
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Checkout neurodamus repository
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
        fetch-depth: 0

    - name: Cache pip
      uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1
        restore-keys: |
          ${{ matrix.os }}-pip-

    - name: Get latest tags
      run: |
        echo "NEURON_BRANCH=${{ inputs.NEURON_BRANCH || github.event.inputs.NEURON_BRANCH || 'master' }}" >> $GITHUB_ENV
        LIBSONATA_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonata/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
        echo "LIBSONATA_BRANCH=${{ inputs.LIBSONATA_BRANCH || github.event.inputs.LIBSONATA_BRANCH || '$LIBSONATA_LATEST' }}" >> $GITHUB_ENV
        LIBSONATA_REPORT_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonatareport/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
        echo "LIBSONATA_REPORT_BRANCH=${{ inputs.LIBSONATA_REPORT_BRANCH || github.event.inputs.LIBSONATA_REPORT_BRANCH || '$LIBSONATA_REPORT_LATEST' }}" >> $GITHUB_ENV
        NEURODAMUS_MODELS_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/neurodamus-models/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')
        echo "NEURODAMUS_MODELS_BRANCH=${{ inputs.NEURODAMUS_MODELS_BRANCH || github.event.inputs.NEURODAMUS_MODELS_BRANCH || '$NEURODAMUS_MODELS_LATEST' }}" >> $GITHUB_ENV

    - name: Get HEAD commit message and look for branches
      run: |
        COMMIT_MESSAGE=$(git log -1 --pretty=%B)
        echo $COMMIT_MESSAGE
        NEURON_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'NEURON_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
        if [[ ! -z $NEURON_BRANCH ]]; then echo "NEURON_BRANCH=$NEURON_BRANCH" >> $GITHUB_ENV; fi
        LIBSONATA_REPORT_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_REPORT_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
        if [[ ! -z $LIBSONATA_REPORT_BRANCH ]]; then echo "LIBSONATA_REPORT_BRANCH=$LIBSONATA_REPORT_BRANCH" >> $GITHUB_ENV; fi
        LIBSONATA_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true)
        if [[ ! -z $LIBSONATA_BRANCH ]]; then echo "LIBSONATA_BRANCH=$LIBSONATA_BRANCH" >> $GITHUB_ENV; fi

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y mpich libmpich-dev libhdf5-mpich-dev hdf5-tools flex libfl-dev bison ninja-build

    - name: Cache python virtual env
      id: cache-venv
      uses: actions/cache@v3
      env:
        cache-name: cache-venv
      with:
        path: venv
        key: ${{ matrix.os }}-libsonata-${{ env.LIBSONATA_BRANCH }}-py${{ matrix.python-version }}

    - name: Upgrade pip and install base Python packages
      if: steps.cache-venv.outputs.cache-hit != 'true'
      run: |
        python -m venv venv
        . ./venv/bin/activate
        python -m pip install --upgrade pip setuptools
        pip install cython numpy wheel pkgconfig

    - name: Add virtual environment to PATH
      run: echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH

    - name: Install libsonata
      if: steps.cache-venv.outputs.cache-hit != 'true'
      run: |
        CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata@${{ env.LIBSONATA_BRANCH }}

    - name: Cache libsonatareport
      id: cache-libsonatareport
      uses: actions/cache@v3
      env:
        cache-name: cache-libsonatareport
      with:
        path: libsonatareport
        key: ${{ matrix.os }}-libsonatareport-${{ env.LIBSONATA_REPORT_BRANCH }}-py${{ matrix.python-version }}

    - name: Install libsonatareport
      if: steps.cache-libsonatareport.outputs.cache-hit != 'true'
      run: |
        git clone --branch="${{ env.LIBSONATA_REPORT_BRANCH}}" https://github.com/BlueBrain/libsonatareport.git --recursive --depth=1
        cd libsonatareport
        mkdir build && cd build
        cmake -DCMAKE_INSTALL_PREFIX=$(pwd)/install -DCMAKE_BUILD_TYPE=Release -DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=ON ..
        cmake --build . --parallel
        cmake --build . --target install

    - name: Cache NEURON
      id: cache-neuron
      uses: actions/cache@v3
      env:
        cache-name: cache-neuron
      with:
        path: nrn
        key: ${{ matrix.os }}-neuron-${{ env.NEURON_BRANCH }}-${{ env.NEURON_COMMIT_ID }}-py${{ matrix.python-version }}

    - name: Install NEURON
      if: steps.cache-neuron.outputs.cache-hit != 'true'
      run: |
        export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
        if [[ ${{ env.NEURON_BRANCH }} == 'master' ]]; then
          git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git
          cd nrn
          git checkout ${{ env.NEURON_COMMIT_ID }}
          cd ..
        else
          git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git --depth=1
        fi
        python -m pip install --upgrade pip -r nrn/nrn_requirements.txt
        cmake -B nrn/build -S nrn -G Ninja \
          -DPYTHON_EXECUTABLE=$(which python) \
          -DCMAKE_INSTALL_PREFIX=$(pwd)/nrn/build/install \
          -DNRN_ENABLE_MPI=ON \
          -DNRN_ENABLE_INTERVIEWS=OFF \
          -DNRN_ENABLE_CORENEURON=ON \
          -DCMAKE_C_COMPILER=gcc \
          -DCMAKE_CXX_COMPILER=g++ \
          -DCORENRN_ENABLE_REPORTING=ON \
          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR
        cmake --build nrn/build --parallel
        cmake --build nrn/build --target install

    - name: Install mpi4py
      if: steps.cache-venv.outputs.cache-hit != 'true'
      run: |
        pip install mpi4py

    - name: Build h5py with the local hdf5
      run: |
        CC="mpicc" HDF5_MPI="ON" HDF5_INCLUDEDIR=/usr/include/hdf5/mpich HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich \
          pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation

    - name: Install neurodamus
      run: |
        pip install .[full]

    - name: Build neocortex models
      run: |
        export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install
        export PATH=$(pwd)/nrn/build/install/bin:$PATH
        # Clone neurodamus-models repository
        git clone --branch=${{ env.NEURODAMUS_MODELS_BRANCH }} https://github.com/BlueBrain/neurodamus-models.git
        # Build neocortex model
        DATADIR=$(python -c "import neurodamus; from pathlib import Path; print(Path(neurodamus.__file__).parent / 'data')")
        cmake -B neurodamus-models/build -S neurodamus-models/ \
          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install \
          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
          -DNEURODAMUS_CORE_DIR=${DATADIR} \
          -DNEURODAMUS_MECHANISMS=neocortex \
          -DNEURODAMUS_NCX_V5=ON
        cmake --build neurodamus-models/build
        cmake --install neurodamus-models/build
        echo "NEURODAMUS_NEOCORTEX_ROOT=$(pwd)/neurodamus-models/install" >> $GITHUB_ENV;

        # Build neocortex-multiscale model
        echo "build neurodamus-neocortex-multiscale model"
        cmake -B neurodamus-models/build_multiscale -S neurodamus-models/ \
          -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install_multiscale \
          -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \
          -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \
          -DNEURODAMUS_CORE_DIR=${DATADIR} \
          -DNEURODAMUS_MECHANISMS=neocortex \
          -DNEURODAMUS_NCX_METABOLISM=ON \
          -DNEURODAMUS_NCX_NGV=ON \
          -DNEURODAMUS_ENABLE_CORENEURON=OFF
        cmake --build neurodamus-models/build_multiscale
        cmake --install neurodamus-models/build_multiscale
        echo "NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT=$(pwd)/neurodamus-models/install_multiscale" >> $GITHUB_ENV;

    - name: load neocortex model
      run: |
        export PYTHONPATH=$(pwd)/nrn/build/install/lib/python:$PYTHONPATH
        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_ROOT/share/neurodamus_neocortex/hoc
        export NEURODAMUS_PYTHON=$(pwd)/neurodamus/data
        export CORENEURONLIB=$NEURODAMUS_NEOCORTEX_ROOT/lib/libcorenrnmech.so
        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_ROOT/lib/libnrnmech.so
        export PATH=$NEURODAMUS_NEOCORTEX_ROOT/bin:$PATH

        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV;
        echo "HOC_LIBRARY_PATH=$HOC_LIBRARY_PATH" >> $GITHUB_ENV;
        echo "CORENEURONLIB=$CORENEURONLIB" >> $GITHUB_ENV;
        echo "NRNMECH_LIB_PATH=$NRNMECH_LIB_PATH" >> $GITHUB_ENV;
        echo "NEURODAMUS_PYTHON=$NEURODAMUS_PYTHON" >> $GITHUB_ENV;

        which special

        # launch simulation with NEURON
        cd tests/simulations/usecase3/
        mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata.json
        ls reporting/*.h5
        # launch simulation with CORENEURON
        mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata_coreneuron.json
        ls reporting_coreneuron/*.h5

    - name: Install test dependencies
      run: |
        pip install -r tests/requirements.txt

    - name: test-unit
      run: |
        pytest tests/unit

    - name: test-integration
      run: |
        pytest -x --forked tests/integration

    - name: test-integration-e2e
      run: |
        pytest -s -x --forked --durations=5 --durations-min=15 tests/integration-e2e

    - name: test-scientific
      run: |
        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific

    - name: test-scientific-ngv
      run: |
        export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/share/neurodamus_neocortex/hoc
        export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/lib/libnrnmech.so
        unset CORENEURONLIB
        pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific-ngv

    - name: live debug session, comment out
      if: failure()
      uses: mxschmitt/action-tmate@v3
      with:
       timeout-minutes: 60  # or any other value you need