slideQuantifyLocal

#!/bin/bash

### Creating display functions
### Setting colouring
NONE='\033[00m'
BOLD='\033[1m'
ITALIC='\033[3m'
OPAQUE='\033[2m'
FLASHING='\033[5m'
UNDERLINE='\033[4m'

RED='\033[01;31m'
GREEN='\033[01;32m'
YELLOW='\033[01;33m'
PURPLE='\033[01;35m'
CYAN='\033[01;36m'
WHITE='\033[01;37m'
### Regarding changing the 'type' of the things printed with 'echo'
### Refer to: 
### - http://askubuntu.com/questions/528928/how-to-do-underline-bold-italic-strikethrough-color-background-and-size-i
### - http://misc.flogisoft.com/bash/tip_colors_and_formatting
### - http://unix.stackexchange.com/questions/37260/change-font-in-echo-command

### echo -e "\033[1mbold\033[0m"
### echo -e "\033[3mitalic\033[0m" ### THIS DOESN'T WORK ON MAC!
### echo -e "\033[4munderline\033[0m"
### echo -e "\033[9mstrikethrough\033[0m"
### echo -e "\033[31mHello World\033[0m"
### echo -e "\x1B[31mHello World\033[0m"

function echobold { #'echobold' is the function name
    echo -e "${BOLD}${1}${NONE}" # this is whatever the function needs to execute, note ${1} is the text for echo
}
function echoitalic { 
    echo -e "${ITALIC}${1}${NONE}" 
}
function echocyan { 
    echo -e "${CYAN}${1}${NONE}" 
}

function echonooption { 
    echo -e "${OPAQUE}${RED}${1}${NONE}"
}

# errors
function echoerrorflash { 
    echo -e "${RED}${BOLD}${FLASHING}${1}${NONE}" 
}
function echoerror { 
    echo -e "${RED}${1}${NONE}"
}

# errors no option
function echoerrornooption { 
    echo -e "${YELLOW}${1}${NONE}"
}
function echoerrorflashnooption { 
    echo -e "${YELLOW}${BOLD}${FLASHING}${1}${NONE}"
}

### MESSAGE FUNCTIONS
script_copyright_message() {
	echo ""
	THISYEAR=$(date +'%Y')
	echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
	echo "+ The MIT License (MIT)                                                                                 +"
	echo "+ Copyright (c) 2016-${THISYEAR} Sander W. van der Laan                                                        +"
	echo "+                                                                                                       +"
	echo "+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and     +"
	echo "+ associated documentation files (the \"Software\"), to deal in the Software without restriction,         +"
	echo "+ including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +"
	echo "+ and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +"
	echo "+ subject to the following conditions:                                                                  +"
	echo "+                                                                                                       +"
	echo "+ The above copyright notice and this permission notice shall be included in all copies or substantial  +"
	echo "+ portions of the Software.                                                                             +"
	echo "+                                                                                                       +"
	echo "+ THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT     +"
	echo "+ NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND                +"
	echo "+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES  +"
	echo "+ OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN   +"
	echo "+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                            +"
	echo "+                                                                                                       +"
	echo "+ Reference: http://opensource.org.                                                                     +"
	echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
}
script_arguments_error() {
	echoerror "$1" # ERROR MESSAGE
	echoerror "- Argument #1  -- name of the stain as it appears in the filenames, e.g. FIBRIN."
	echoerror "- Argument #2  -- path_to CellProfiler pipeline, e.g. FIBRIN.cppipe."
	echoerror "- Argument #3  -- path_to working directory, i.e. where all the image-subdirectories are."
	echoerror "- Argument #4  -- starting letters/characters of the image-subdirectories, e.g. IMG or AE or AAA."
	echoerror "- Argument #5  -- output filename where the CellProfiler results are stored, e.g. Image.txt (delimiter is assumed '_')."
	echoerror ""
	echoerror "An example command would be: slideQuantify [arg1: STAIN] [arg2: path_to_cellprofiler_pipeline] [arg3: path_to_working_directory] [arg4: IMG or AE or AAA] [arg5: Image.txt]"
	echoerror "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
  	# The wrong arguments are passed, so we'll exit the script now!
  	exit 1
}

echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echobold "                               slideQuantify"
echo ""
echoitalic "* Written by  : Sander W. van der Laan; Tim Bezemer; Tim van de Kerkhof"
echoitalic "* E-mail      : s.w.vanderlaan-2@umcutrecht.nl"
echoitalic "* Last update : 2019-07-03"
echoitalic "* Version     : 1.1.3-beta"
echo ""
echoitalic "* Description : This script will start the quantification for a given stain"
echoitalic "                in a given project directory."
echo ""
echocyan "NOTE: this is the macOS (10.14.5+) version for local use."
echo ""
echo "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"

echo ""
### REQUIRED | GENERALS	
STAIN="$1" # Depends on arg1
PIPELINE="$2" # Depends on arg2
PROJECTDIR="$3" # Depends on arg3
FILESTART="$4" # Depends on arg4
OUTPUTFILENAME="$5" # Depends on arg5

### Set slideToolKit DIRECTORY
SLIDETOOLKITDIR="~/git/swvanderlaan/slideToolKit"

### START of if-else statement for the number of command-line arguments passed ###
if [[ $# -lt 5 ]]; then 
	echo "Oh, computer says no! Number of arguments found "$#"."
	script_arguments_error "You must supply correct (number of) arguments when running *** slideQuantify ***!"
		
else
	echobold "Cleaning up house, first."
	rm -rfv $FILESTART*/magick-tmp

	ENTROPY_THRESHOLD=210
	HOLD_JIDS=()
	INDEX=0
	BATCH_SIZE=5000
	UNIQUE_LABEL="V3"

	### FOR DEBUG
	### DEBUGLIST="AE1686.T07-11206.CD3.20141216 AE1927.T08-12829.CD3.20141216 AE337.T06-19497.CD3.20141218 AE1827.T09-14098.CD3 AE2191.UMC.CD3.20141216 AE921.T05-16731.CD3.20150901"
	### for SLIDE_NUM in $DEBUGLIST; do
	for SLIDE_NUM in $FILESTART*; do
		sleep 1
		THIS_SLIDE=${SLIDE_NUM/\//}
		THIS_SLIDEMASK=${THIS_SLIDE}_slideMask_${UNIQUE_LABEL}
		THIS_SLIDE2TILES=${THIS_SLIDE}_slide2Tiles_${UNIQUE_LABEL}
		THIS_SLIDENORMALIZE=${THIS_SLIDE}_slideNormalize_${UNIQUE_LABEL}
		THIS_CELLPROFILER=${THIS_SLIDE}_CellProfiler_${UNIQUE_LABEL}
		THIS_WRAPUP=${THIS_SLIDE}_WrapUp_${UNIQUE_LABEL}

		HOLD_JID_INDEX=`expr $INDEX % $BATCH_SIZE`

		# process batches in smaller chunks
		if [ "$INDEX" -lt "$BATCH_SIZE" ]; then
			HOLD_JID_STRING=""
		else
			HOLD_JID_STRING="-hold_jid ${HOLD_JIDS[$HOLD_JID_INDEX]}"
		fi
		HOLD_JIDS[$HOLD_JID_INDEX]=$THIS_CELLPROFILER
	
		cd $PROJECTDIR/$SLIDE_NUM

		echo ""
		echobold "Processing images of file [ $SLIDE_NUM ] for staining [ $STAIN ]."
	
		if [ -f results.txt ]; then 
			echoitalic "Checking if this file was processed before ....."
			cd ..; 
			INDEX=$((INDEX+1)); 
			echo "..... Analysis was already done!"; 
			continue; 
		fi
		echo "..... Processing file for the first time."
		
		### slideEMask for macro, and NDPISplit when necessary
		echo ""
		echoitalic "* slideEMask, which will mask, create a macro, and do NDPISplit when necessary. [ $THIS_SLIDEMASK ]"
		
		### NOTE: this does not work locally on macOS
		### if [[ -f *mask.png ]]; then 
		### 	echo "..... Entropy mask already exists."
		### 	continue; 
		### fi;
		mkdir -pv magick-tmp
			export MAGICK_TMPDIR=$(pwd)/magick-tmp
			export TMPDIR=$(pwd)/magick-tmp
		if [ -f *.ndpi ]; then
			echo "The image-file is a NDPI and will be converted to .tif before masking."
			if [ -f *.ndpi ]; then  ndpisplit -x40 -z0 *.ndpi; fi
			slideMask --layer 0 -f *.tif;
		
		elif [ -f *.tif ]; then 
			echo "The image-file is a (NDPI-converted) .tif."
			slideMask --layer 0 -f *.tif;
		
		elif [ -f *.TIF ]; then 
			echo "The image-file is a .TIF."
			slideMask --layer 7 -f *.TIF;
		
		else
			echoerrorflash \"*** ERROR *** Something is rotten in the City of Gotham; most likely a typo. Double back, please. 
			[image-extension not recognized, should be 'ndpi', 'tif' or 'TIF' ]\"	
			exit 1 
		fi
		
		echo "..... Applying entropy filter over [*.macro.png]."
		slideEMask -f *macro.png -t $ENTROPY_THRESHOLD
		echo "..... Removing temporary directory."
		rm -rfv magick-tmp

		### FOR DEBUG
		### echo $THIS_SLIDEMASK
		### echo $HOLD_JID_STRING
	
		### slide2Tiles
		echo ""
		echoitalic "* slide2Tiles to create tiles from images. [ $THIS_SLIDE2TILES ]"
		
		### NOTE: this does not work locally on macOS
		### if [ -d *.tiles ]; then 
		### 	echo "..... Tiles directory already exists."
		### 	continue; 
		### fi;

		mkdir -pv magick-tmp
		export MAGICK_TMPDIR=$(pwd)/magick-tmp
		export TMPDIR=$(pwd)/magick-tmp

		if [ -f *.ndpi ]; then
			echo "The image-file is a NDPI and will be converted to .tif before tiling."
			slide2Tiles --layer 0 -f *x40*.tif -m *emask.png;
		
		elif [ -f *.tif ]; then
			echo "The image-file is a (NDPI-converted) .tif."
			slide2Tiles --layer 0 -f *.tif -m *.emask.png;
		
		elif [ -f *.TIF ]; then
			echo "The image-file is a .TIF."
			slide2Tiles --layer 7 -f *.TIF -m *.emask.png;
		
		else
			echoerrorflash \"*** ERROR *** Something is rotten in the City of Gotham; most likely a typo. Double back, please. 
			[image-extension not recognized, should be 'ndpi', 'tif' or 'TIF' ]\"	
			exit 1 
		fi
		echo "..... Removing temporary directory."
		rm -rfv magick-tmp

		### slideNormalize
		echo ""
		echoitalic "* slideNormalize for normalization of images. [ $THIS_SLIDENORMALIZE ]."
		
		### NOTE: this does not work locally on macOS
		### if [ -f files2cp.txt ]; then 
		### 	echo "Normalization was already applied."
		### 	continue; 
		### fi;
		if [ ! -d *.tiles ]; then
			(>&2 echo "*** ERROR *** No tiles to process.")
			exit; 
		fi
		
		### DEBUG
		### echo "...Checking existence of tiles."
		### ls $PROJECTDIR/$SLIDE_NUM/*.tiles/;
		cd $PROJECTDIR/$SLIDE_NUM/*.tiles/;

		mkdir -pv magick-tmp
			export MAGICK_TMPDIR=$(pwd)/magick-tmp
			export TMPDIR=$(pwd)/magick-tmp
		for MASKED_IMAGE_TILE in *.png; do
			echo "...Processing tile [ $MASKED_IMAGE_TILE ]"
			echo "... - applying normalization ..."
			slideNormalize $MASKED_IMAGE_TILE
			
			echo "... - masking the normalized image ..."
			slideEMask -c -f $MASKED_IMAGE_TILE -t $ENTROPY_THRESHOLD
			rm -v $MASKED_IMAGE_TILE;
		done
		rm -rfv magick-tmp
		cd ..
# 		echo "...Collecting all normalized and masked tiles in a file for CellProfiler."
# 		ls -d -1 $(pwd)/*tiles/*normalized* > files2cp.txt
# 		ls -d -1 $(pwd)/*tiles/ENTROPY* >> files2cp.txt
# 	
# 		### CellProfiler
# 		### This does not work on macOS Mojave 10.14.5 because we need vigra.
# 		### However installing this comes with issues: 
# 		### https://github.com/ukoethe/vigra/issues/468
# 		echo ""
# 		echoitalic "* CellProfiler to analyze images. [ $THIS_CELLPROFILER ]"
# 		
# 		if [ -z "$(ls -A cp_output)" ]; then 
# 			mkdir -pv cp_output
# 			echo "...Running CellProfiler using $PIPELINE for $STAIN stained samples."
# 			cellprofiler -c -r -p $PIPELINE --file-list files2cp.txt -o cp_output/;
# 
# 		else
# 			echo "CellProfiler was already run, or at least there is a 'cp_output'-directory."
# 			
# 		fi
# 
# 		### REMOVE THIS BREAK FOR PRODUCTION!
# 		### When this break is active, only the first item 
# 		### in the list of items to be processed will be processed.
# 		INDEX=$((INDEX+1))
# # 		break;	
# 
# 		### Wrap Up
# 		echo ""
# 		echoitalic "* WrapUp the work we have done, and clean house. [ $THIS_WRAPUP ] "
# # 		
# # 		# Obsolete code
# # 		# 1. we randomly grap 10 masked & tiled PNG-files to keep for post-analysis check >> OUT-COMMENTED
# # 		# 2. we get the data (in STAIN_Image.txt) for each study number (SLIDE_NUM) in 'cp_output' using >> OUT-COMMENTED
# # 		#    Colsums.R
# # 		# 3. we remove tiles, and other files we don't need
# # 		
# 		ls cp_output/*.png | shuf -n $(expr $(ls cp_output/*.png | wc -l) - 10) | xargs rm;
# 		echo 'SLIDE_NUM $STAIN' > results.txt;
# 		cd cp_output;
# 		SLIDE_NUM=$(basename "$PWD") | cut -d'.' -f1
# 		echo $SLIDE_NUM $(Rscript $SLIDETOOLKITDIR/utilities/Colsums.R $STAIN $OUTPUTFILENAME) >> ../results.txt;
# 		gzip -v ../results.txt;
# 		cd ..
# 		#rm -rfv *tiles/;
# 		if [ -f *.ndpi ]; then 
# 			echo "Removing intermediate tif- & png-files converted from NDPI-files.";
# 			#rm -v *x40*.tif; 
# 			#rm -v *x40*.png; 
# 		fi;
# 			echo "Removing list of files to process.";
# 			#rm -v files2cp.txt;
# 
		cd ..

		INDEX=$((INDEX+1))
		if [ "$INDEX" -eq "$BATCH_SIZE" ]; then 
			break;
		fi
		
	done
	
### END of if-else statement for the number of command-line arguments passed ###
fi

script_copyright_message