slideQuantifyDev

#!/bin/bash

### Creating display functions
### Setting colouring
NONE='\033[00m'
BOLD='\033[1m'
ITALIC='\033[3m'
OPAQUE='\033[2m'
FLASHING='\033[5m'
UNDERLINE='\033[4m'

RED='\033[01;31m'
GREEN='\033[01;32m'
YELLOW='\033[01;33m'
PURPLE='\033[01;35m'
CYAN='\033[01;36m'
WHITE='\033[01;37m'
### Regarding changing the 'type' of the things printed with 'echo'
### Refer to: 
### - http://askubuntu.com/questions/528928/how-to-do-underline-bold-italic-strikethrough-color-background-and-size-i
### - http://misc.flogisoft.com/bash/tip_colors_and_formatting
### - http://unix.stackexchange.com/questions/37260/change-font-in-echo-command

### echo -e "\033[1mbold\033[0m"
### echo -e "\033[3mitalic\033[0m" ### THIS DOESN'T WORK ON MAC!
### echo -e "\033[4munderline\033[0m"
### echo -e "\033[9mstrikethrough\033[0m"
### echo -e "\033[31mHello World\033[0m"
### echo -e "\x1B[31mHello World\033[0m"

function echobold { #'echobold' is the function name
    echo -e "${BOLD}${1}${NONE}" # this is whatever the function needs to execute, note ${1} is the text for echo
}
function echoitalic { 
    echo -e "${ITALIC}${1}${NONE}" 
}
function echocyan { 
    echo -e "${CYAN}${1}${NONE}" 
}

function echonooption { 
    echo -e "${OPAQUE}${RED}${1}${NONE}"
}

# errors
function echoerrorflash { 
    echo -e "${RED}${BOLD}${FLASHING}${1}${NONE}" 
}
function echoerror { 
    echo -e "${RED}${1}${NONE}"
}

# errors no option
function echoerrornooption { 
    echo -e "${YELLOW}${1}${NONE}"
}
function echoerrorflashnooption { 
    echo -e "${YELLOW}${BOLD}${FLASHING}${1}${NONE}"
}

### MESSAGE FUNCTIONS
script_copyright_message() {
	echo ""
	THISYEAR=$(date +'%Y')
	echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
	echo "+ The MIT License (MIT)                                                                                 +"
	echo "+ Copyright (c) 2016-${THISYEAR} Sander W. van der Laan                                                        +"
	echo "+                                                                                                       +"
	echo "+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and     +"
	echo "+ associated documentation files (the \"Software\"), to deal in the Software without restriction,         +"
	echo "+ including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +"
	echo "+ and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +"
	echo "+ subject to the following conditions:                                                                  +"
	echo "+                                                                                                       +"
	echo "+ The above copyright notice and this permission notice shall be included in all copies or substantial  +"
	echo "+ portions of the Software.                                                                             +"
	echo "+                                                                                                       +"
	echo "+ THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT     +"
	echo "+ NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND                +"
	echo "+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES  +"
	echo "+ OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN   +"
	echo "+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                            +"
	echo "+                                                                                                       +"
	echo "+ Reference: http://opensource.org.                                                                     +"
	echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
}
script_arguments_error() {
	echoerror "$1" # ERROR MESSAGE
	echoerror "- Argument #1  -- name of the stain as it appears in the filenames, e.g. FIBRIN."
	echoerror "- Argument #2  -- path_to CellProfiler pipeline, e.g. FIBRIN.cppipe."
	echoerror "- Argument #3  -- path_to working directory, i.e. where all the image-subdirectories are."
	echoerror "- Argument #4  -- starting letters/characters of the image-subdirectories, e.g. IMG or AE or AAA."
	echoerror "- Argument #5  -- output filename where the CellProfiler results are stored, e.g. Image.txt (delimiter is assumed '_')."
	echoerror "- Argument #6  -- email address to send aborted/error jobs too, e.g. y.name@umcutrecht.nl."
	echoerror "- Argument #7  -- by default emails are send when jobs are aborted/error, you can change that here, e.g. you can choose: b=begin of job; e=end of job; a=abort of job; s=suspended job; n=no mail is send."
	echoerror "- Argument #8  -- eMask threshold. A smaller number is less stringent, best results are obtained using, e.g. '210'."
	echoerror ""
	echoerror "An example command would be: slideQuantify [arg1: STAIN] [arg2: path_to_cellprofiler_pipeline] [arg3: path_to_working_directory] [arg4: IMG or AE or AAA] [arg5: Image.txt] [arg6: y.name@umcutrecht.nl] [arg7: a] [arg8: 210]"
	echoerror "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
  	# The wrong arguments are passed, so we'll exit the script now!
  	exit 1
}

echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echobold "                         slideQuantify -- Development"
echo ""
echoitalic "* Written by  : Sander W. van der Laan; Tim Bezemer; Tim van de Kerkhof"
echoitalic "* E-mail      : s.w.vanderlaan-2@umcutrecht.nl"
echoitalic "* Last update : 2019-07-23"
echoitalic "* Version     : 1.1.8"
echo ""
echoitalic "* Description : This script will start the quantification for a given stain"
echoitalic "                in a given project directory. Use for development, no"
echoitalic "                intermediate files are removed."
echo ""
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"

echo ""
### REQUIRED | GENERALS	
STAIN="$1" # Depends on arg1
PIPELINE="$2" # Depends on arg2
PROJECTDIR="$3" # Depends on arg3
FILESTART="$4" # Depends on arg4
OUTPUTFILENAME="$5" # Depends on arg5
EMAIL="$6" # Depends on arg6
MAILSETTING="$7" # Depends on arg7
EMASKTHRESHOLD="$8" # Depends on arg8

QJOB_MEM_SLIDEMASK="100G"
QJOB_TIME_SLIDEMASK="02:30:00"

QJOB_MEM_SLIDE2TILES="150G"
QJOB_TIME_SLIDE2TILES="04:30:00"

QJOB_MEM_SLIDENORM="5G"
QJOB_TIME_SLIDENORM="02:00:00"

QJOB_MEM_CP="10G"
QJOB_TIME_CP="09:00:00"

# Set slideToolKit DIRECTORY
SLIDETOOLKITDIR="/hpc/local/CentOS7/dhl_ec/software/slideToolKit"

### START of if-else statement for the number of command-line arguments passed ###
if [[ $# -lt 8 ]]; then 
	echo "Oh, computer says no! Number of arguments found \"$#\"."
	script_arguments_error "You must supply correct (number of) arguments when running *** slideQuantify ***!"
		
else
	echobold "Cleaning up house, first."
	mkdir -pv "$PROJECTDIR"/logs
	mkdir -pv "$PROJECTDIR"/errors
	rm -rfv "$FILESTART"*/magick-tmp

	ENTROPY_THRESHOLD="$EMASKTHRESHOLD" # should be argument
	HOLD_JIDS=()
	INDEX=0
	BATCH_SIZE=5000 # should be argument, but how exactly does this work?
	UNIQUE_LABEL="V3" # should be argument

	### FOR DEBUG
	### DEBUGLIST="AE999.FIBRIN AE99.T02-18575.A.FIBRIN AE997.T05-12460.FIBRIN"
	### for SLIDE_NUM in $DEBUGLIST; do
	for SLIDE_NUM in "$FILESTART"*; do
		sleep 1
		THIS_SLIDE=${SLIDE_NUM/\//}
		THIS_SLIDEMASK=${THIS_SLIDE}_slideMask_${UNIQUE_LABEL}
		THIS_SLIDE2TILES=${THIS_SLIDE}_slide2Tiles_${UNIQUE_LABEL}
		THIS_SLIDENORMALIZE=${THIS_SLIDE}_slideNormalize_${UNIQUE_LABEL}
		THIS_CELLPROFILER=${THIS_SLIDE}_CellProfiler_${UNIQUE_LABEL}
		THIS_WRAPUP=${THIS_SLIDE}_WrapUp_${UNIQUE_LABEL}

		HOLD_JID_INDEX=$(expr ${INDEX} % ${BATCH_SIZE})

		# process batches in smaller chunks
		if [ "$INDEX" -lt "$BATCH_SIZE" ]; then
			HOLD_JID_STRING=""
		else
			HOLD_JID_STRING="-hold_jid ${HOLD_JIDS[$HOLD_JID_INDEX]}"
		fi
		HOLD_JIDS[$HOLD_JID_INDEX]=$THIS_CELLPROFILER
	
		cd "$SLIDE_NUM" || exit
		echo ""
		echobold "Processing images of file [ ${SLIDE_NUM} ] for staining [ ${STAIN} ] -- at index [ ${INDEX} ] with total batch size [ ${BATCH_SIZE} ]."
	
		if [[ -s cp_output/"$STAIN"_Image.csv.gz ]]; then 
			echoitalic "Checking if this file was processed before; cp_output/${STAIN}_Image.csv.gz should be present ....."
			ls -lh cp_output/*.gz
			cd .. ; 
			INDEX=$((INDEX+1)); 
			echo "..... Analysis was already done!"; 
			continue; 
		fi
		
		echo "..... Processing file for the first time. Submitting job for: "
		
		rm -rfv "$PROJECTDIR"/logs/"$SLIDE_NUM"*
		rm -rfv "$PROJECTDIR"/errors/"$SLIDE_NUM"*

	### slideEMask for macro, and NDPISplit when necessary
	echo ""
	echoitalic "* slideEMask, which will mask, create a macro, and do NDPISplit when necessary."
	
	echo "#if [ -s *mask.png ]; then exit; fi;
	module load slideToolkit
	module load ndpitools
	mkdir -pv magick-tmp
		export MAGICK_TMPDIR=$(pwd)/magick-tmp
		export TMPDIR=$(pwd)/magick-tmp
	if [ -f *.ndpi ]; then
		echo \"The image-file is a NDPI and will be converted to .tif before masking.\"
		if [ -f *.ndpi ]; then  ndpisplit -x40 -z0 *.ndpi; fi
		slideMask --layer 0 -f *.tif;
	
	elif [ -f *.tif ]; then 
		echo \"The image-file was NDPI and converted to .tif.\"
		slideMask --layer 0 -f *.tif;
	
	elif [ -f *.TIF ]; then 
		echo \"The image-file is a .TIF.\"
		slideMask --layer 3 -f *.TIF;
	
	else
		echoerrorflash \"*** ERROR *** Something is rotten in the City of Gotham; most likely a typo. Double back, please. 
		[image-extension not recognized, should be 'ndpi', 'tif' or 'TIF' ]\"	
		exit 1 
	fi
	slideEMask -f *macro.png -t $ENTROPY_THRESHOLD
	rm -rfv magick-tmp
	" | qsub -cwd -N ${THIS_SLIDEMASK} -e ${PROJECTDIR}/errors/ -o ${PROJECTDIR}/logs/ ${HOLD_JID_STRING} -l h_vmem=${QJOB_MEM_SLIDEMASK} -l h_rt=${QJOB_TIME_SLIDEMASK} -m ${MAILSETTING} -M ${EMAIL}
	### FOR DEBUG
	### echo $THIS_SLIDEMASK
	### echo $HOLD_JID_STRING

	### slide2Tiles
	echo ""
	echoitalic "* slide2Tiles to create tiles from images."
	
	echo "if [ -d *.tiles ]; then exit; fi;
	module load ndpitools;
	module load slideToolkit;
	mkdir -pv magick-tmp
	export MAGICK_TMPDIR=$(pwd)/magick-tmp
	export TMPDIR=$(pwd)/magick-tmp

	if [ -f *.ndpi ]; then
		echo \"The image-file is a NDPI and will be converted to .tif before tiling.\"
		slide2Tiles --layer 0 -f *x40*.tif -m *emask.png;
	
	elif [ -f *.tif ]; then
		echo \"The image-file was NDPI and converted to .tif.\"
		slide2Tiles --layer 0 -f *.tif -m *.emask.png;
	
	elif [ -f *.TIF ]; then
		echo \"The image-file is a .TIF.\"
		slide2Tiles --layer 3 -f *.TIF -m *.emask.png;
	
	else
		echoerrorflash \"*** ERROR *** Something is rotten in the City of Gotham; most likely a typo. Double back, please. 
		[image-extension not recognized, should be 'ndpi', 'tif' or 'TIF' ]\"	
		exit 1 
	fi
	rm -rfv magick-tmp
	" | qsub -cwd -N ${THIS_SLIDE2TILES} -e ${PROJECTDIR}/errors/ -o ${PROJECTDIR}/logs/ -hold_jid ${THIS_SLIDEMASK} -l h_vmem=${QJOB_MEM_SLIDE2TILES} -l h_rt=${QJOB_TIME_SLIDE2TILES} -m ${MAILSETTING} -M ${EMAIL}

	### slideNormalize
	echo ""
	echoitalic "* slideNormalize for normalization of images."
	
	echo "if [ -s files2cp.txt ]; then exit; fi;
	if [ ! -d *.tiles ]; then
		(>&2 echo "*** ERROR *** No tiles to process. ")
		exit; 
	fi
	cd *.tiles/;
	module load slideNormalize;
	module load slideToolkit;
	mkdir -pv magick-tmp
		export MAGICK_TMPDIR=$(pwd)/magick-tmp
		export TMPDIR=$(pwd)/magick-tmp
	for f in *.png; do
		echo \"...Processing tile \$f\";
		slideNormalize \$f;
		slideEMask -c -f \$f -t $ENTROPY_THRESHOLD
		rm -v \$f;
	done
	rm -rfv magick-tmp
	cd ..
	ls -d -1 \$PWD/*tiles/*normalized* > files2cp.txt;
	ls -d -1 \$PWD/*tiles/ENTROPY* >> files2cp.txt;
	" | qsub -cwd -N ${THIS_SLIDENORMALIZE} -e ${PROJECTDIR}/errors/ -o ${PROJECTDIR}/logs/ -hold_jid ${THIS_SLIDE2TILES} -l h_vmem=${QJOB_MEM_SLIDENORM} -l h_rt=${QJOB_TIME_SLIDENORM} -m ${MAILSETTING} -M ${EMAIL}

	### CellProfiler
	echo ""
	echoitalic "* CellProfiler to analyze images."
	
	echo "if [ -d cp_output ]; then exit; fi;
	module load anaconda
	mkdir -pv cp_output
	cellprofiler -c -r -p $PIPELINE --file-list files2cp.txt -o cp_output/;
	" | qsub -cwd -N ${THIS_CELLPROFILER} -e ${PROJECTDIR}/errors/ -o ${PROJECTDIR}/logs/ -hold_jid ${THIS_SLIDENORMALIZE} -l h_vmem=${QJOB_MEM_CP} -l h_rt=${QJOB_TIME_CP} -m ${MAILSETTING} -M ${EMAIL}

	### REMOVE THIS BREAK FOR PRODUCTION!
	### When this break is active, only the first item 
	### in the list of items to be processed will be processed.
# 		INDEX=$((INDEX+1))
# 		break;	

	### Wrap Up
	echo ""
	echoitalic "* WrapUp the work we have done, and clean house."
	
	# Obsolete code
	# 1. we randomly grap 10 masked & tiled PNG-files to keep for post-analysis check >> OUT-COMMENTED
	# 2. we get the data (in STAIN_Image.txt) for each study number (SLIDE_NUM) in 'cp_output' using >> OUT-COMMENTED
	#    Colsums.R
	# 3. we remove tiles, and other files we don't need
	
	echo "
	#ls cp_output/*.png | shuf -n \$(expr \$(ls cp_output/*.png | wc -l) - 10) | xargs rm;
	echo 'SLIDE_NUM $STAIN' > results.txt;
	cd cp_output;
	SLIDE_NUM=\$(basename \"$PWD\") | cut -d'.' -f1
	echo $SLIDE_NUM \$(Rscript $SLIDETOOLKITDIR/utilities/Colsums.R $STAIN $OUTPUTFILENAME) >> ../results.txt;
	cd ..
	#rm -rfv *tiles/;
	if [ -f *.ndpi ]; then 
		echo \"Removing intermediate tif- & png-files converted from NDPI-files.\";
		#rm -v *x40*.tif; 
		#rm -v *x40*.png; 
		fi;
		echo \"Removing list of files to process.\";
		#rm -v files2cp.txt;
		echo \"Gzipping result files.\";
		gzip -vf cp_output/${STAIN}*.gct;
		gzip -vf cp_output/${STAIN}*.csv;
	" | qsub -cwd -N ${THIS_WRAPUP} -e ${PROJECTDIR}/errors/ -o ${PROJECTDIR}/logs/ -hold_jid ${THIS_CELLPROFILER} -m ${MAILSETTING} -M ${EMAIL}

		cd ..

		INDEX=$((INDEX+1))
		if [ "$INDEX" -eq "$BATCH_SIZE" ]; then 
			break; 
		fi;
	done


### END of if-else statement for the number of command-line arguments passed ###
fi

script_copyright_message