Skip to content

Commit

Permalink
Merge branch 'dwi-dev' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
perronea committed Apr 19, 2020
2 parents 7f89549 + e35dbdb commit ead64e0
Show file tree
Hide file tree
Showing 8 changed files with 766 additions and 401 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Clone this repository and save it somewhere on the Linux system you want to do A
1. [MathWorks MATLAB Runtime Environment (MRE) version 9.1 (R2016b)](https://www.mathworks.com/products/compiler/matlab-runtime.html)
1. [cbedetti Dcm2Bids](https://github.com/cbedetti/Dcm2Bids) (`export` into your BASH `PATH` variable)
1. [Rorden Lab dcm2niix](https://github.com/rordenlab/dcm2niix) (`export` into your BASH `PATH` variable)
1. [dcmdump](https://dicom.offis.de/dcmtk.php.en) (`export` into your BASH `PATH` variable)
1. [zlib's pigz-2.4](https://zlib.net/pigz) (`export` into your BASH `PATH` variable)
1. Docker (see documentation for [Docker Community Edition for Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/))
1. [FMRIB Software Library (FSL) v5.0](https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FslInstallation)
Expand Down
159 changes: 122 additions & 37 deletions abcd2bids.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@
CORRECT_JSONS = os.path.join(PWD, "src", "correct_jsons.py")
DOWNLOAD_FOLDER = os.path.join(PWD, "raw")
NDA_AWS_TOKEN_MAKER = os.path.join(PWD, "src", "nda_aws_token_maker.py")
SERIES_TABLE_PARSER = os.path.join(PWD, "src", "good_bad_series_parser.py")
SPREADSHEET_DOWNLOAD = os.path.join(PWD, "spreadsheets",
"ABCD_good_and_bad_series_table.csv")

SERIES_TABLE_PARSER = os.path.join(PWD, "src", "aws_downloader.py")
SPREADSHEET_DOWNLOAD = os.path.join(PWD, "spreadsheets", "ABCD_good_and_bad_series_table.csv")
SPREADSHEET_QC = os.path.join(PWD, "spreadsheets", "abcd_fastqc01.txt")
TEMP_FILES_DIR = os.path.join(PWD, "temp")
UNPACK_AND_SETUP = os.path.join(PWD, "src", "unpack_and_setup.sh")
UNPACKED_FOLDER = os.path.join(PWD, "data")
MODALITIES = ['anat', 'func', 'dwi']


def main():
Expand Down Expand Up @@ -196,6 +197,30 @@ def get_cli_args():
"spreadsheet.".format(SPREADSHEET_QC))
)

# Optional: Subject list
parser.add_argument(
"-l",
"--subject-list",
dest="subject_list",
type=validate_readable_file,
required=True,
help=("Path to a .txt file containing a list of subjects to download. "
"The default is to download all available subjects.")
)

# Optional: Modalities
parser.add_argument(
"-m",
"--modalities",
choices=MODALITIES,
nargs="+",
dest="modalities",
default=MODALITIES,
help=("List of the imaging modalities that should be downloaded for "
"each subject. The default is to download all modalities. "
"The possible selections are {}".format(MODALITIES))
)

# Optional: During unpack_and_setup, remove unprocessed data
parser.add_argument(
"-rm",
Expand Down Expand Up @@ -243,6 +268,17 @@ def get_cli_args():
"then the user will be prompted for their NDA password.")
)

parser.add_argument(
"-z",
"--docker-cmd",
type=str,
dest="docker_cmd",
default=None,
help=("A necessary docker command replacement on HPCs like "
"the one at OHSU, which has it's own special wrapper for"
"docker for security reasons. Example: '/opt/acc/sbin/exadocker'")
)

# Parse, validate, and return all CLI args
return validate_cli_args(parser.parse_args(), parser)

Expand Down Expand Up @@ -573,8 +609,12 @@ def download_nda_data(cli_args):
with downloaded NDA data.
:return: N/A
"""
subprocess.check_call(("python3", SERIES_TABLE_PARSER, cli_args.download,
SPREADSHEET_DOWNLOAD))
subprocess.check_call(("python3", "--version"))
subprocess.check_call(("python3",
SERIES_TABLE_PARSER,
"--download-dir", cli_args.download,
"--subject-list", cli_args.subject_list,
"--modalities", ','.join(cli_args.modalities)))


def unpack_and_setup(cli_args):
Expand All @@ -586,34 +626,74 @@ def unpack_and_setup(cli_args):
--output, --download, --temp, and --remove.
:return: N/A
"""
for subject in os.scandir(cli_args.download):
if subject.is_dir():
for session_dir in os.scandir(subject.path):
if session_dir.is_dir():
for tgz in os.scandir(session_dir.path):
if tgz:

# Get session ID from some (arbitrary) .tgz file in
# session folder
session_name = tgz.name.split("_")[1]

# Unpack/setup the data for this subject/session
subprocess.check_call((UNPACK_AND_SETUP,
subject.name,
"ses-" + session_name,
session_dir.path,
cli_args.output,
cli_args.temp,
cli_args.fsl_dir,
cli_args.mre_dir))

# If user said to, delete all the raw downloaded
# files for each subject after that subject's data
# has been converted and copied
if cli_args.remove:
shutil.rmtree(os.path.join(cli_args.download,
subject.name))
break

if args.subject_list:
f = open(args.subject_list, 'r')
x = f.readlines()
f.close
subject_list = [sub.strip() for sub in x]
for subject in subject_list:
subject_dir = os.path.join(args.download, subject)
if os.path.isdir(subject_dir):
for session_dir in os.scandir(subject_dir):
if session_dir.is_dir():
for tgz in os.scandir(session_dir.path):
if tgz:

# Get session ID from some (arbitrary) .tgz file in
# session folder
session_name = tgz.name.split("_")[1]

# Unpack/setup the data for this subject/session
subprocess.check_call((
UNPACK_AND_SETUP,
subject,
"ses-" + session_name,
session_dir.path,
args.output,
args.temp,
args.fsl_dir,
args.mre_dir
))

# If user said to, delete all the raw downloaded
# files for each subject after that subject's data
# has been converted and copied
if args.remove:
shutil.rmtree(os.path.join(args.download,
subject))
break
else:
for subject in os.scandir(args.download):
if subject.is_dir():
for session_dir in os.scandir(subject.path):
if session_dir.is_dir():
for tgz in os.scandir(session_dir.path):
if tgz:

# Get session ID from some (arbitrary) .tgz file in
# session folder
session_name = tgz.name.split("_")[1]

# Unpack/setup the data for this subject/session
subprocess.check_call((
UNPACK_AND_SETUP,
subject.name,
"ses-" + session_name,
session_dir.path,
args.output,
args.temp,
args.fsl_dir,
args.mre_dir
))

# If user said to, delete all the raw downloaded
# files for each subject after that subject's data
# has been converted and copied
if args.remove:
shutil.rmtree(os.path.join(args.download,
subject.name))
break


def correct_jsons(cli_args):
Expand Down Expand Up @@ -655,12 +735,17 @@ def validate_bids(cli_args):
:return: N/A
"""
try:
subprocess.check_call(("docker", "run", "-ti", "--rm", "-v",
cli_args.output + ":/data:ro", "bids/validator",
"/data"))
if cli_args.docker_cmd:
subprocess.check_call(('sudo', cli_args.docker_cmd, "run", "-ti", "--rm", "-v",
cli_args.output + ":/data:ro", "bids/validator",
"/data"))
else:
subprocess.check_call(("docker", "run", "-ti", "--rm", "-v",
cli_args.output + ":/data:ro", "bids/validator",
"/data"))
except subprocess.CalledProcessError:
print("Error: BIDS validation failed.")


if __name__ == '__main__':
main()
main()
76 changes: 71 additions & 5 deletions abcd_dcm2bids.conf
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,83 @@
{
"descriptions": [
{
"dataType": "dwi",
"modalityLabel": "dwi",
"criteria": {
"SeriesDescription": "ABCD-DTI_SIEMENS_mosaic_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "dwi",
"modalityLabel": "dwi",
"criteria": {
"SeriesDescription": "ABCD-DTI_PHILIPS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "dwi",
"modalityLabel": "dwi",
"criteria": {
"SeriesDescription": "ABCD-DTI_GE_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "acq-dwi_dir-AP",
"intendedFor": 0,
"criteria": {
"SeriesDescription": "ABCD-Diffusion-FM-AP_SIEMENS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "acq-dwi_dir-AP",
"intendedFor": 1,
"criteria": {
"SeriesDescription": "ABCD-Diffusion-FM-AP_PHILIPS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "acq-dwi_dir-AP",
"intendedFor": 2,
"criteria": {
"SeriesDescription": "ABCD-Diffusion-FM_GE_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "acq-dwi_dir-PA",
"intendedFor": 0,
"criteria": {
"SeriesDescription": "ABCD-Diffusion-FM-PA_SIEMENS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "acq-dwi_dir-PA",
"intendedFor": 1,
"criteria": {
"SeriesDescription": "ABCD-Diffusion-FM-PA_PHILIPS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "dir-AP",
"customLabels": "acq-func_dir-AP",
"criteria": {
"SeriesDescription": "ABCD-fMRI-FM-AP_SIEMENS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "dir-PA",
"customLabels": "acq-func_dir-PA",
"criteria": {
"SeriesDescription": "ABCD-fMRI-FM-PA_SIEMENS_original_(baseline_year_1_arm_1)"
}
Expand Down Expand Up @@ -81,15 +147,15 @@
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "dir-PA",
"customLabels": "acq-func_dir-PA",
"criteria": {
"SeriesDescription": "ABCD-fMRI-FM-PA_PHILIPS_original_(baseline_year_1_arm_1)"
}
},
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "dir-AP",
"customLabels": "acq-func_dir-AP",
"criteria": {
"SeriesDescription": "ABCD-fMRI-FM-AP_PHILIPS_original_(baseline_year_1_arm_1)"
}
Expand Down Expand Up @@ -143,7 +209,7 @@
{
"dataType": "fmap",
"modalityLabel": "epi",
"customLabels": "dir-both",
"customLabels": "acq-func_dir-both",
"criteria": {
"SeriesDescription": "ABCD-fMRI-FM_GE_original_(baseline_year_1_arm_1)"
}
Expand Down
Loading

0 comments on commit ead64e0

Please sign in to comment.