Merge branch 'main' into joss

Author: sinhaharsh

.github/workflows/continuous-integration.yml

@@ -8,6 +8,11 @@ on:
         branches:
             - main
 
+    pull_request:
+        branches:
+            - main
+
+
 jobs:
     py:
         runs-on: ${{ matrix.os }}
@@ -26,14 +31,15 @@ jobs:
             -   name: Install dependencies
                 run: |
                     python -m pip install --upgrade pip
-                    if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi
                     pip install .
+                    pip install .[test]
             -   name: Lint with flake8
                 run: |
                     make lint
             -   name: Test with pytest
+                working-directory: ${{ runner.temp }}
                 run: |
-                    pytest
+                    pytest --pyargs MRdataset
 
     py312:
         runs-on: ${{ matrix.os }}
@@ -61,5 +67,6 @@ jobs:
                 run: |
                     make lint
             -   name: Test with pytest
+                working-directory: ${{ runner.temp }}
                 run: |
-                    pytest
+                    pytest --pyargs MRdataset

MRdataset/__init__.py

@@ -5,6 +5,7 @@
 
 # __version__ = '0.1.0'
 import logging
+import sys
 
 from MRdataset.config import configure_logger
 
@@ -16,10 +17,18 @@
 from MRdataset.dicom_utils import is_dicom_file
 from MRdataset.utils import valid_dirs
 from MRdataset.base import BaseDataset
+from MRdataset.bids import BidsDataset
 from MRdataset.dicom import DicomDataset
 
 try:
     from MRdataset._version import __version__
 except ImportError:
-    raise ImportError('It seems MRdataset is not installed correctly. Use pip '
-                      ' to install it first.')
+    if sys.version_info < (3, 8):
+        from importlib_metadata import version
+    else:
+        from importlib.metadata import version
+
+    try:
+        __version__ = version('MRdataset')
+    except Exception:
+        __version__ = "unknown"

MRdataset/bids.py

@@ -1,13 +1,158 @@
-# from MRdataset.base import BaseDataset
-# from abc import ABC
-#
-#
-# class BidsDataset(BaseDataset, ABC):
-#     """Class to represent a BIDS dataset"""
-#
-#     def __init__(self):
-#         super().__init__()
-#         pass
-#
-#     def load(self):
-#         pass
+from abc import ABC
+from pathlib import Path
+from re import search
+
+from MRdataset import logger
+from MRdataset.base import BaseDataset
+from MRdataset.config import VALID_BIDS_DATATYPES, SUPPORTED_BIDS_DATATYPES
+from MRdataset.dicom_utils import is_bids_file
+from MRdataset.utils import folders_with_min_files, valid_dirs, read_json
+from protocol import BidsImagingSequence
+
+
+class BidsDataset(BaseDataset, ABC):
+    """
+    Class to represent a BIDS dataset. It is a subclass of BaseDataset.
+    It gathers data from JSON files.
+
+    Parameters
+    ----------
+    data_source : str or List[str]
+        The path to the dataset.
+    pattern : str
+        The pattern to match for JSON files.
+    name : str
+        The name of the dataset.
+    config_path : str
+        The path to the config file.
+    verbose : bool
+        Whether to print verbose output on console.
+    ds_format : str
+        The format of the dataset. One of ['dicom', 'bids'].
+    """
+
+    def __init__(self, data_source, pattern="*.json",
+                 name='BidsDataset',
+                 config_path=None,
+                 verbose=False,
+                 output_dir=None,
+                 min_count=1,
+                 **kwargs):
+
+        super().__init__(data_source=data_source, name=name, ds_format='bids')
+        self.data_source = valid_dirs(data_source)
+        self.pattern = pattern
+        self.config_path = config_path
+        self.verbose = verbose
+        self.config_dict = None
+        self.min_count = min_count
+
+        try:
+            self.output_dir = Path(output_dir)
+        except TypeError as exc:
+            logger.error('Output directory not valid. Got: {output_dir}')
+            raise exc
+
+        self.output_dir.mkdir(exist_ok=True, parents=True)
+
+        # read the config file
+        try:
+            self.config_dict = read_json(Path(self.config_path))
+        except (FileNotFoundError, ValueError) as e:
+            logger.error(f'Unable to read config file {self.config_path}')
+            raise e
+
+        self.includes = self.config_dict.get('include_sequence', {})
+        self.include_nifti_headers = self.includes.get('nifti_header', False)
+
+    def load(self):
+        """
+        Default method to load the dataset. It iterates over all the folders
+        in the data_source and finds subfolders with at least min_count files
+        matching the pattern. It then processes each subfolder and adds the
+        sequence to the dataset.
+        """
+
+        for directory in self.data_source:
+            # find all sub-folders with at least min_count files matching the
+            # pattern
+            subfolders = folders_with_min_files(directory, self.pattern,
+                                                self.min_count)
+            for folder in subfolders:
+                # process each folder
+                sequences = self._process(folder)
+                for seq in sequences:
+                    self.add(subject_id=seq.subject_id,
+                             session_id=seq.session_id,
+                             run_id=seq.run_id,
+                             seq_id=seq.name, seq=seq)
+
+    def _filter_json_files(self, folder):
+        """Filters the JSON files from the folder."""
+        json_files = sorted(folder.glob(self.pattern))
+        valid_bids_files = list(filter(is_bids_file, json_files))
+        if not valid_bids_files:
+            logger.info(f'No valid BIDS files found in {folder}')
+            return []
+        return valid_bids_files
+
+    def _process(self, folder):
+        """Processes the folder and returns a list of sequences."""
+        json_files = self._filter_json_files(folder)
+        sequences = []
+        last_id = 0
+        for i, file in enumerate(json_files):
+            try:
+                seq = BidsImagingSequence(bidsfile=file, path=folder)
+            except (ValueError, IOError) as exc:
+                logger.error(f'Error processing {file}. Skipping it. Got {exc}')
+                continue
+
+            name = file.parent.name
+            if name not in VALID_BIDS_DATATYPES:
+                logger.error(f'Invalid datatype found: {name}. Skipping it')
+                return sequences
+
+            subject_id = file.parents[2].name
+            session_id = file.parents[1].name
+            if 'sub' in session_id:
+                logger.info(f"Sessions don't exist: {session_id}.")
+                subject_id = session_id
+                session_id = 'ses-01'
+
+            # None of the datasets we processed (over 20) had run information,
+            # even though BIDS allows it. So we just use run-0x for all of them.
+            run_id, last_id = self.get_run_id(file, last_id)
+            seq.set_session_info(subject_id=subject_id,
+                                 session_id=session_id,
+                                 run_id=run_id,
+                                 name=name)
+            if seq.is_valid():
+                sequences.append(seq)
+            else:
+                if name not in SUPPORTED_BIDS_DATATYPES:
+                    logger.error(f'MRdataset primarily supports '
+                                 f'{SUPPORTED_BIDS_DATATYPES}'
+                                 f'It seems the parameters in '
+                                 f'this sequence are invalid or '
+                                 f'not supported yet. Skipping it.')
+        return sequences
+
+    @staticmethod
+    def get_run_id(filename, last_id):
+        """
+        Use regex to extract run id from filename.
+        Example filename : sub-01_ses-imagery01_task-imagery_run-01_bold.json
+        """
+        # Regular expression pattern
+        pattern = r'run-[^_]+'
+        # Extracting substring using regex
+        match = search(pattern, str(filename))
+
+        if match:
+            run_id = match.group(0)
+            new_id_num = int(run_id.split('-')[-1])
+        else:
+            new_id_num = last_id + 1
+            run_id = f'run-{str(new_id_num).zfill(2)}'
+        return run_id, new_id_num

MRdataset/cli.py

@@ -50,6 +50,14 @@ def parse_args():
                        f'Using default config file - {str(args.config)}')
 
     if args.output_dir:
+        output_dir = Path(args.output_dir)
+        if not output_dir.is_dir():
+            try:
+                output_dir.mkdir(parents=True, exist_ok=True)
+            except OSError as exc:
+                logger.error('Unable to create folder: {output_dir}')
+                raise exc
+
         if not is_writable(args.output_dir):
             raise OSError('Expected a writable directory for --output_dir '
                           f'argument, Got {args.output_dir}')

MRdataset/common.py

@@ -4,6 +4,7 @@
 
 from MRdataset import logger
 from MRdataset.base import BaseDataset
+from MRdataset.bids import BidsDataset
 from MRdataset.config import VALID_DATASET_FORMATS
 from MRdataset.dicom import DicomDataset
 from MRdataset.utils import random_name, check_mrds_extension
@@ -126,6 +127,8 @@ def find_dataset_using_ds_format(dataset_ds_format: str):
     # Import the module "{ds_format}_dataset.py"
     if dataset_ds_format == 'dicom':
         dataset_class = DicomDataset
+    elif dataset_ds_format == 'bids':
+        dataset_class = BidsDataset
     else:
         raise NotImplementedError(
             f'Dataset ds_format {dataset_ds_format} is not implemented. '

MRdataset/config.py

@@ -16,7 +16,7 @@
 VALID_BIDS_EXTENSIONS = ['.json', '.nii', '.nii.gz']
 
 
-VALID_DATATYPES = [
+VALID_BIDS_DATATYPES = [
     'anat',
     'beh',
     'dwi',
@@ -30,8 +30,8 @@
     'pet'
 ]
 
+SUPPORTED_BIDS_DATATYPES = ['func', 'anat', 'dwi', 'fmap']
 
-# Suppress duplicated warnings
 
 class MRException(Exception):
     """
@@ -59,7 +59,7 @@ def __init__(self):
                          '--data_source. Got 0 DICOM/JSON files.')
 
 
-def configure_logger(log, output_dir, mode='w', level='WARNING'):
+def configure_logger(log, output_dir, mode='w', level='ERROR'):
     """
     Initiate log files.
 

MRdataset/dicom.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Tuple, List
 
-from protocol import ImagingSequence
+from protocol import DicomImagingSequence
 from pydicom import dcmread
 from pydicom.errors import InvalidDicomError
 
@@ -58,17 +58,17 @@ def __init__(self,
                  name='DicomDataset',
                  config_path=None,
                  verbose=False,
-                 ds_format='dicom',
                  output_dir=None,
+                 min_count=1,
                  **kwargs):
         """constructor"""
 
         super().__init__(data_source=data_source, name=name,
-                         ds_format=ds_format)
+                         ds_format='dicom')
         self.data_source = valid_dirs(data_source)
         self.pattern = pattern
         # TODO: Add option to change min_count passing it as an argument
-        self.min_count = 1  # min slice count to be considered a volume
+        self.min_count = min_count  # min slice count to be considered a volume
         self.verbose = verbose
         self.config_path = config_path
         self.config_dict = None
@@ -215,7 +215,7 @@ def _process_slice_collection(self, folder):
         example, EchoTime and Echonumber for multi-echo sequences.
 
         It then processes the divergent slices to find the varying parameters
-        and updates the protocol.ImagingSequence object.
+        and updates the protocol.DicomImagingSequence object.
 
         Parameters
         ----------
@@ -262,14 +262,14 @@ def _process_slice_collection(self, folder):
             # Note that we cannot use enumerate and idx ==0 here, because we
             #   may have to skip some slices
             if len(divergent_slices) == 0:
-                first_slice = ImagingSequence(dicom=dicom, path=folder)
+                first_slice = DicomImagingSequence(dicom=dicom, path=folder)
                 # We collect the first slice as a reference to compare
                 #   other slices with, although it is not divergent in
                 #   its true sense
                 divergent_slices.append(first_slice)
 
             else:
-                cur_slice = ImagingSequence(dicom=dicom, path=folder)
+                cur_slice = DicomImagingSequence(dicom=dicom, path=folder)
 
                 # check if the session info is same
                 # Session info includes subject_id, session_id, run_id
@@ -329,7 +329,7 @@ def _process_echo_times(self, divergent_slices: List) -> Tuple:
         Parameters
         ----------
         divergent_slices : list
-            ImagingSequence objects with divergent parameters
+            DicomImagingSequence objects with divergent parameters
 
         Returns
         -------