01-generate-htc-scripts.sh

#!/usr/bin/env bash

# ###
#
#  A script aims at generating HTC-BLAST execution scripts to be
#  used on a scheduler-based computational infrastructure.
#
#  Originally designed to evaluate BLAST on DATARMOR, the 
#  supercomputer from Ifremer.
#
#  Use:
#
#   ./01-generate-htc-scripts.sh
#
#  Before running this script, please edit the template file:
#
#   htc_blast_template.pbs
#
#  @author Patrick G. Durand, Laure Quintric, Ifremer, Nov 2017
# 

# File containing list of generated scripts
# It can be used to submit BLAST job by order
# (as generated by this script)
SCRIPTLISTFILE="script_list.txt"
[ -e $SCRIPTLISTFILE ] && rm $SCRIPTLISTFILE
touch $SCRIPTLISTFILE

# File containg the BLAST template
FILE="htc_blast_template.txt"

# Nb. "chunk:cores" to test
CORES=("1:28" "2:28" "3:28" "4:28" "10:28")

# BLAST comparisons to test:
#  P:P = protein query vs protein bank
#  M:N = nucl query (megablast) vs nucl bank
#  N:N = nucl query (blastn) vs nucl bank
#  P:N = protein query vs. nucl bank
#  (P,M,N and more  are defined in the template file).
COMPS=("P:P" "M:N" "N:N" "N:P")

# Memory to use (unit: Gb)
MEM=115

# Walltime to use (Unit: hours)
WTIME=48

nbscript=0

# For each type of BLAST comparison...
for COMP in "${COMPS[@]}" ; do
  QTYPE=${COMP%:*}
  STYPE=${COMP#*:}
  # ... we will generate a BLAST script using a 
  #     a particular nb cores (note: we always start
  #     from upper to lower values)
  for (( j = ${#CORES[@]}-1 ; j >=0 ; j-- )) ; do
    CHUNK_CORE="${CORES[j]}"
    CHUNK=${CHUNK_CORE%:*}
    CORE=${CHUNK_CORE#*:}
    FCORE=$((CHUNK*CORE))
    # generate a BLAST script file name
    FSCRIPT=${QTYPE}-${STYPE}-${MEM}gb-${FCORE}c.pbs
    echo "Generating $FSCRIPT ..."
    [ -e $FSCRIPT ] && rm $FSCRIPT
    touch $FSCRIPT
    chmod +x $FSCRIPT
    echo "$FSCRIPT" >> $SCRIPTLISTFILE
    nbscript=$((nbscript+1))
    # Read the template line by line and replace some 
    # variables (defined in the template) by their 
    # values as set in this script 
    while IFS= read -r line
    do
      line=${line//@CORE@/$CORE}
      line=${line//@CHUNK@/$CHUNK}
      line=${line//@FCORE@/$FCORE}
      line=${line//@MEM@/$MEM}
      line=${line//@QTYPE@/$QTYPE}
      line=${line//@STYPE@/$STYPE}
      line=${line//@WTIME@/$WTIME}
      echo "$line" >> $FSCRIPT
    done < "$FILE"
  done
done

echo ""
echo "$nbscript scripts generated"
echo "You can now use '02-submit-scripts.sh' to submit jobs to PBS"