add BIDS Conversion pipeline

.gitignore

@@ -164,3 +164,9 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+
+#logs
+logs/
+
+#ignore poetry.lock
+poetry.lock

.pre-commit-config.yaml

@@ -17,6 +17,8 @@ repos:
   hooks:
   - id: mypy
     args: [--ignore-missing-imports]
+    additional_dependencies:
+    - types-PyYAML
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
   rev: v2.12.0
   hooks:

README.md

@@ -10,25 +10,47 @@ Welcome to the ```sailsprep``` repo! This is a Python repo for doing incredible
 
 **Caution:**: this package is still under development and may change rapidly over the next few weeks.
 
+This will convert the raw video into BIDS format in a clean fashion.
 ## Features
 - A few
 - Cool
 - Things
 - These may include a wonderful CLI interface.
 
 ## Installation
+To manage dependencies, this project uses Poetry. Make sure you've got poetry installed.
+On Engaging, you need to first run at the root of the repo :
+```
+module load miniforge
+pip install poetry
+poetry install
+```
+
+The BIDS-conversion tool of sailsprep requires FFmpeg ≥ 6.0 compiled with the vidstab library.
+Because FFmpeg compiled with vidstab is not a Python package, it must be installed separately.
+You'll need to run (outside any environment):
+
+```
+cd ~
+wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+tar -xJf ffmpeg-release-amd64-static.tar.xz
+mv ffmpeg-*-static ffmpeg_static
+export PATH="$HOME/ffmpeg_static:$PATH"
+
+```
+
 Get the newest development version via:
 
 ```sh
 pip install git+https://github.com/sensein/sailsprep.git
 ```
-
 ## Quick start
-```Python
-from sailsprep.app import hello_world
 
-hello_world()
-```
+Tools developped in sailsprep
+|Tool|Documentation|
+|----|--------------|
+|BIDS-conversion| [link to documentation](docs/BIDS_convertor.md)
+
 
 ## Contributing
 We welcome contributions from the community! Before getting started, please review our [**CONTRIBUTING.md**](https://github.com/sensein/sailsprep/blob/main/CONTRIBUTING.md).

configs/config_bids_convertor.yaml

@@ -0,0 +1,17 @@
+# Video Processing Configuration
+
+# Input data
+annotation_file: /orcd/data/satra/002/datasets/SAILS/data4analysis/Video Rating Data/SAILS_RATINGS_ALL_DEDUPLICATED_NotForFinalAnalyses_2025.10.csv
+video_root: /orcd/data/satra/002/datasets/SAILS/Phase_III_Videos/Videos_from_external
+asd_status: /orcd/data/satra/002/datasets/SAILS/data4analysis/ASD_Status.xlsx
+
+# Output data
+output_dir: /orcd/scratch/bcs/001/sensein/sails/BIDS_data
+
+# Video processing parameters
+target_resolution: 1280x720
+target_framerate: 30
+
+# Derived directory names (optional — can be built dynamically)
+final_bids_root: final_bids-dataset
+derivatives_subdir: derivatives/preprocessed

docs/BIDS_convertor.md

@@ -0,0 +1,52 @@
+## BIDS Format
+
+For reproducibility, organization, and practicality, sailsprep converts its raw data into the BIDS (Brain Imaging Data Structure) format.
+BIDS is a community-driven standard for organizing, naming, and describing neuroimaging and related data (e.g., EEG, fMRI, MEG, behavioral, physiological data, etc.).
+
+During the BIDS conversion pipeline, the raw domestic videos are preprocessed to be standardized, denoised, and reformatted.
+Relevant metadata and annotations necessary for downstream analysis are also extracted at this stage.
+
+## Structure
+
+The final BIDS dataset follows the structure below:
+```graphql
+├── sub-ID1         # Contains raw videos in BIDS format
+│   ├── ses-01          # Videos between 12 and 16 months
+│   │   └── beh                 # Behavioral data
+│   │        ├── sub-ID1_ses-01_task-A_run-01_beh.mp4   # Standardized raw video
+│   │        ├── sub-ID1_ses-01_task-A_run-01_beh.tsv   # Manual annotations
+│   │        └── sub-ID1_ses-01_task-A_run-01_beh.json  # Info on standardization
+│   └── ses-02          # Videos between 34 and 38 months
+│       └── beh
+├── derivatives
+│   └── preprocessed # Contains stabilized, denoised, standardized videos
+│       ├── sub-ID1
+│       │   ├── ses-01
+│       │   │   └── beh
+│       │   │        ├── sub-ID1_ses-01_task-A_run-01_audio.json              # Audio extraction info
+│       │   │        ├── sub-ID1_ses-01_task-A_run-01_audio.wav               # Extracted audio
+│       │   │        ├── sub-ID1_ses-01_task-A_run-01_desc-processed.json     # Video preprocessing info
+│       │   │        └── sub-ID1_ses-01_task-A_run-01_desc-processed_beh.mp4  # Preprocessed video
+│       │   └── ses-02
+│       └── sub-ID2
+├── README.md                   # Explains dataset structure and content
+├── participants.tsv            # Participant information (e.g., ASD status)
+├── participants.json           # Metadata for participants.tsv
+└── dataset_description.json    # BIDS dataset description (name, version, etc.)
+```
+## Execution
+
+To verify that FFmpeg is correctly installed (cf [README.md](../README.md)) and at least version 6.0, run:
+
+```
+ffmpeg -version
+```
+
+You’ll need to submit the conversion job on Engaging using sbatch.
+Make sure you are in the root directory of the repository.
+
+We provide SLURM submission scripts for convenience — simply run the following commands (with the miniforge module deactivated to ensure the correct FFmpeg version is used):
+```
+jid=$(sbatch --parsable jobs/run_bids_convertor.sh)
+sbatch --dependency=afterok:$jid jobs/merge_cleanup.sh
+```

jobs/merge_cleanup.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#SBATCH --job-name=merge_cleanup
+#SBATCH --output=logs/merge_cleanup_%j.out
+#SBATCH --error=logs/merge_cleanup_%j.err
+#SBATCH --time=01:00:00
+#SBATCH --mem=2G
+
+# Clean up old logs before running
+echo "Cleaning up old logs..."
+if [ -d logs ]; then
+    find logs -mindepth 1 ! -name ".gitkeep" \
+         ! -name "merge_cleanup_${SLURM_JOB_ID}.out" \
+         ! -name "merge_cleanup_${SLURM_JOB_ID}.err" -delete
+fi
+
+OUTPUT_DIR=$(poetry run python -c "import yaml, sys; print(yaml.safe_load(open('configs/config_bids_convertor.yaml'))['output_dir'])")
+MERGED_DIR="$OUTPUT_DIR"
+
+mkdir -p "$MERGED_DIR"
+
+echo "Merging logs from numbered folders under $OUTPUT_DIR"
+echo "Started at $(date)"
+
+merged_processed="$MERGED_DIR/all_processed.json"
+merged_failed="$MERGED_DIR/all_failed.json"
+
+# Create empty lists if not exist
+echo "[]" > "$merged_processed"
+echo "[]" > "$merged_failed"
+
+# Load jq (if not already available)
+module load jq 2>/dev/null || true
+
+for folder in "$OUTPUT_DIR"/*/; do
+    foldername=$(basename "$folder")
+
+    if [[ "$foldername" =~ ^[0-9]+$ ]]; then
+        echo "Merging from folder: $foldername"
+        if [[ -f "$folder/processing_log.json" ]]; then
+            tmpfile=$(mktemp)
+            jq -s 'add' "$merged_processed" "$folder/processing_log.json" > "$tmpfile" && mv "$tmpfile" "$merged_processed"
+        fi
+        if [[ -f "$folder/not_processed.json" ]]; then
+            tmpfile=$(mktemp)
+            jq -s 'add' "$merged_failed" "$folder/not_processed.json" > "$tmpfile" && mv "$tmpfile" "$merged_failed"
+        fi
+    fi
+done
+
+echo "Merged logs saved in: $MERGED_DIR"
+echo "Now cleaning up numbered folders..."
+
+# Delete only folders with numeric names (avoid final_bids-dataset)
+for folder in "$OUTPUT_DIR"/*/; do
+    foldername=$(basename "$folder")
+    if [[ "$foldername" =~ ^[0-9]+$ ]]; then
+        echo "Deleting temporary folder: $foldername"
+        rm -rf "$folder"
+    else
+        echo "Skipping non-numeric folder: $foldername"
+    fi
+done
+
+echo "Cleanup complete at $(date)"
+
+# --- Run final Python merge ---
+echo "Running final Python merge and participant file creation..."
+poetry run python -c "from sailsprep.BIDS_convertor import merge_subjects, create_participants_file; merge_subjects(); create_participants_file()"
+echo "Final BIDS merge and participant file creation complete ✅"

jobs/run_bids_convertor.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#SBATCH --job-name=bids_processing
+#SBATCH --partition=mit_normal
+#SBATCH --array=0-18
+#SBATCH --output=logs/bids_%A_%a.out
+#SBATCH --error=logs/bids_%A_%a.err
+#SBATCH --mem=5G
+#SBATCH --time=10:00:00
+#SBATCH --cpus-per-task=5
+
+mkdir -p logs
+
+# --- Determine project root robustly ---
+if [ -n "$SLURM_SUBMIT_DIR" ]; then
+    cd "$SLURM_SUBMIT_DIR" || { echo "❌ Cannot cd to SLURM_SUBMIT_DIR=$SLURM_SUBMIT_DIR"; exit 1; }
+else
+    SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+    cd "$SCRIPT_DIR/.." || { echo "❌ Cannot cd to project root"; exit 1; }
+fi
+
+echo "Running from project root: $(pwd)"
+export PYTHONUNBUFFERED=1
+
+ffmpeg -version || echo "⚠️ FFmpeg not available"
+
+# --- Poetry setup ---
+if ! poetry env info --path &> /dev/null; then
+    echo "Creating Poetry environment..."
+    poetry install || { echo "❌ Poetry install failed"; exit 1; }
+fi
+
+ENV_PATH=$(poetry env info --path)
+source "$ENV_PATH/bin/activate" || { echo "❌ Failed to activate Poetry environment"; exit 1; }
+
+echo "Using Python from: $(which python)"
+echo "Task ID: ${SLURM_ARRAY_TASK_ID}"
+echo "Starting BIDS conversion at $(date)"
+
+python -m sailsprep.BIDS_convertor "$SLURM_ARRAY_TASK_ID" "$SLURM_ARRAY_TASK_MAX"
+
+echo "Finished at $(date)"

pyproject.toml

@@ -35,7 +35,11 @@ requires-poetry = ">=2.0"
 version = "0.0.0"
 
 [tool.poetry.dependencies]
-click = "~=8.3"
+click = "~=8.2"
+pandas = "^2.3.3"
+opencv-python = "^4.12.0.88"
+openpyxl = "^3.1.5"
+types-pyyaml = "^6.0.12.20250915"
 
 [tool.poetry.group.dev]
 optional = true