From 5b9cd05a28a89b07e9d4f861ed5470b1dcb49460 Mon Sep 17 00:00:00 2001 From: Tahir D'Mello Date: Mon, 17 Jun 2024 18:20:47 -0700 Subject: [PATCH 1/5] Automated commit: processed repository cutandrun --- .dockerignore | 6 + ...sfulltest.yml => awsfulltest.yml.disabled} | 0 .../{awstest.yml => awstest.yml.disabled} | 0 .../{branch.yml => branch.yml.disabled} | 0 .github/workflows/{ci.yml => ci.yml.disabled} | 0 .../{clean-up.yml => clean-up.yml.disabled} | 0 ...ine.yml => download_pipeline.yml.disabled} | 0 ...x-linting.yml => fix-linting.yml.disabled} | 0 .../{linting.yml => linting.yml.disabled} | 0 ...mment.yml => linting_comment.yml.disabled} | 0 ...yml => release-announcements.yml.disabled} | 0 .../docker-build-logs.txt | 109 ++++ .latch/bin/nextflow | 458 +++++++++++++++++ .latch/config | 1 + .latch/nextflow_version | 1 + Dockerfile | 46 ++ latch.config | 9 + latch_metadata/__init__.py | 23 + latch_metadata/parameters.py | 480 ++++++++++++++++++ version | 1 + wf/entrypoint.py | 212 ++++++++ 21 files changed, 1346 insertions(+) create mode 100644 .dockerignore rename .github/workflows/{awsfulltest.yml => awsfulltest.yml.disabled} (100%) rename .github/workflows/{awstest.yml => awstest.yml.disabled} (100%) rename .github/workflows/{branch.yml => branch.yml.disabled} (100%) rename .github/workflows/{ci.yml => ci.yml.disabled} (100%) rename .github/workflows/{clean-up.yml => clean-up.yml.disabled} (100%) rename .github/workflows/{download_pipeline.yml => download_pipeline.yml.disabled} (100%) rename .github/workflows/{fix-linting.yml => fix-linting.yml.disabled} (100%) rename .github/workflows/{linting.yml => linting.yml.disabled} (100%) rename .github/workflows/{linting_comment.yml => linting_comment.yml.disabled} (100%) rename .github/workflows/{release-announcements.yml => release-announcements.yml.disabled} (100%) create mode 100644 .latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt create mode 100755 .latch/bin/nextflow create mode 100644 .latch/config create mode 100644 .latch/nextflow_version create mode 100644 Dockerfile create mode 100644 latch.config create mode 100644 latch_metadata/__init__.py create mode 100644 latch_metadata/parameters.py create mode 100644 version create mode 100644 wf/entrypoint.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1b1dda5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +.nextflow* +.nextflow.log* +work/ +results/ diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml.disabled similarity index 100% rename from .github/workflows/awsfulltest.yml rename to .github/workflows/awsfulltest.yml.disabled diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml.disabled similarity index 100% rename from .github/workflows/awstest.yml rename to .github/workflows/awstest.yml.disabled diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml.disabled similarity index 100% rename from .github/workflows/branch.yml rename to .github/workflows/branch.yml.disabled diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml.disabled similarity index 100% rename from .github/workflows/ci.yml rename to .github/workflows/ci.yml.disabled diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml.disabled similarity index 100% rename from .github/workflows/clean-up.yml rename to .github/workflows/clean-up.yml.disabled diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml.disabled similarity index 100% rename from .github/workflows/download_pipeline.yml rename to .github/workflows/download_pipeline.yml.disabled diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml.disabled similarity index 100% rename from .github/workflows/fix-linting.yml rename to .github/workflows/fix-linting.yml.disabled diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml.disabled similarity index 100% rename from .github/workflows/linting.yml rename to .github/workflows/linting.yml.disabled diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml.disabled similarity index 100% rename from .github/workflows/linting_comment.yml rename to .github/workflows/linting_comment.yml.disabled diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml.disabled similarity index 100% rename from .github/workflows/release-announcements.yml rename to .github/workflows/release-announcements.yml.disabled diff --git a/.latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt b/.latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt new file mode 100644 index 00000000..a1c2a33e --- /dev/null +++ b/.latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt @@ -0,0 +1,109 @@ +Step 1/15 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main + + + ---> f23c37efcd53 + +Step 2/15 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> dd16c2d4f334 + +Step 3/15 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> aaada0c60239 + +Step 4/15 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> d6869bc03deb + +Step 5/15 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> ae2f7ca65880 + +Step 6/15 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> f029b3f36a86 + +Step 7/15 : run pip install latch==2.46.6 + + + ---> Using cache + + ---> be1350f11537 + +Step 8/15 : run mkdir /opt/latch + + + ---> Using cache + + ---> e8dd4419614b + +Step 9/15 : run apt-get update && apt-get install -y default-jre-headless + + + ---> Using cache + + ---> 447cbd90cbc9 + +Step 10/15 : copy . /root/ + + + ---> 3f3e4ac9e1cf + +Step 11/15 : run ln -s /root/.latch/bin/nextflow /root/nextflow + + + ---> Running in c15dac84f040 + +ln -s /root/.latch/bin/nextflow /root/nextflow + + ---> c24b956d7b18 + +Step 12/15 : run ln -s /root/.latch/.nextflow /root/.nextflow + + + ---> Running in 72a50362b6e5 + +ln -s /root/.latch/.nextflow /root/.nextflow + + ---> b64ac576009f + +Step 13/15 : arg tag + + + ---> Running in 235d45699fc3 + + ---> c1de3bbb49b5 + +Step 14/15 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 7662e8e183e9 + + ---> 11282b33ea19 + +Step 15/15 : workdir /root + + + ---> Running in b851b30539aa + + ---> a2635d95d5dd + +Successfully built a2635d95d5dd + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35597_nf_nf_core_cutandrun:0.1.0-88a084 + diff --git a/.latch/bin/nextflow b/.latch/bin/nextflow new file mode 100755 index 00000000..7906c2aa --- /dev/null +++ b/.latch/bin/nextflow @@ -0,0 +1,458 @@ +#!/usr/bin/env bash +# +# Copyright 2013-2023, Seqera Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[ "$NXF_DEBUG" == 'x' ]] && set -x +NXF_VER=${NXF_VER:-'23.11.0-edge'} +NXF_ORG=${NXF_ORG:-'nextflow-io'} +NXF_HOME=${NXF_HOME:-$HOME/.nextflow} +NXF_PROT=${NXF_PROT:-'https'} +NXF_BASE=${NXF_BASE:-$NXF_PROT://www.nextflow.io/releases} +NXF_TEMP=${NXF_TEMP:-$TMPDIR} +NXF_DIST=${NXF_DIST:-$NXF_HOME/framework} +NXF_CLI="$0 $@" +NXF_CLI_OPTS=${NXF_CLI_OPTS:-} +NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} + +export NXF_CLI +export NXF_ORG +export NXF_HOME + +if [[ $TERM && $TERM != 'dumb' ]]; then +if command -v tput &>/dev/null; then +GREEN=$(tput setaf 2; tput bold) +YELLOW=$(tput setaf 3) +RED=$(tput setaf 1) +NORMAL=$(tput sgr0) +fi +fi + +function echo_red() { + >&2 echo -e "$RED$*$NORMAL" +} + +function echo_green() { + echo -e "$GREEN$*$NORMAL" +} + +function echo_yellow() { + >&2 echo -e "$YELLOW$*$NORMAL" +} + +function die() { + echo_red "$*" + exit 1 +} + +function get_abs_filename() { + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +function get() { + if command -v curl &>/dev/null; then + GET="curl -fsSL '$1' -o '$2'" + elif command -v wget &>/dev/null; then + GET="wget '$1' -O '$2' >/dev/null 2>&1" + else + echo_red "ERROR: Cannot find 'curl' nor 'wget' utility -- please install one of them" + exit 1 + fi + + printf "Downloading nextflow dependencies. It may require a few seconds, please wait .. " + eval $GET; status=$? + printf "\r\033[K" + if [ $status -ne 0 ]; then + echo_red "ERROR: Cannot download nextflow required file -- make sure you can connect to the internet" + echo "" + echo "Alternatively you can try to download this file:" + echo " $1" + echo "" + echo "and save it as:" + echo " ${3:-$2}" + echo "" + exit 1 + fi +} + +function get_ver() { + if command -v curl &>/dev/null; then + curl -fsSL "$1" + elif command -v wget &>/dev/null; then + wget "$1" >/dev/null 2>&1 + else + echo_red "ERROR: Cannot find 'curl' nor 'wget' utility -- please install one of them" + exit 1 + fi +} + +function make_temp() { + local base=${NXF_TEMP:=$PWD} + if [ "$(uname)" = 'Darwin' ]; then mktemp "${base}/nxf-tmp.XXXXXX" || exit $? + else mktemp -t nxf-tmp.XXXXXX -p "${base}" || exit $? + fi +} + +function resolve_link() { + [[ ! -f $1 ]] && exit 1 + if command -v realpath &>/dev/null; then + realpath "$1" + elif command -v readlink &>/dev/null; then + local target="$1" + cd "$(dirname "$target")"; target="$(basename "$target")" + while [ -L "$target" ]; do + target="$(readlink "$target")" + cd "$(dirname "$target")"; target="$(basename "$target")" + done + echo "$(cd "$(dirname "$target")"; pwd -P)/$target" + else + echo_yellow "WARN: Neither \`realpath\` nor \`readlink\` command can be found" + exit 1 + fi +} + +function current_ver() { + [[ $NXF_EDGE == 1 || $NXF_VER == *"-edge" ]] && printf 'edge' || printf 'latest' +} + +function install() { + local tmpfile=$(make_temp) + local version=$(set +u; [[ $NXF_VER ]] && printf "v$NXF_VER" || current_ver) + local action="a=${2:-default}" + get "$NXF_BASE/$version/nextflow?$action" "$tmpfile" "$1" || exit $? + mv "$tmpfile" "$1" || exit $? + chmod +x "$1" || exit $? + bash "$1" -download || exit $? + echo '' + echo -e $'Nextflow installation completed. Please note:' + echo -e $'- the executable file `nextflow` has been created in the folder:' $(dirname $1) + if [[ ! "$PATH" =~ (^|:)"$(dirname $1)"(:|$) ]]; then + echo -e $'- you may complete the installation by moving it to a directory in your $PATH' + fi + echo '' +} + +function check_latest() { + [[ $cmd != run ]] && return 0 + [[ $NXF_OFFLINE == true || $NXF_DISABLE_CHECK_LATEST == true ]] && return 0 + local latest=$(get_ver "$NXF_BASE/$(current_ver)/version?current=$NXF_VER") + if [[ -n "$latest" && "$latest" != $NXF_VER ]]; then + echo_yellow "Nextflow $latest is available - Please consider updating your version to it" + fi +} + +function launch_nextflow() { + # the launch command line + local cmdline=() + # remove leading and trailing double-quotes + for x in "${launcher[@]}"; do + x="${x%\"}" + x="${x#\"}" + cmdline+=("$x") + done + + if [[ "$bg" ]]; then + local pid_file="${NXF_PID_FILE:-.nextflow.pid}" + cmdline+=("${args[@]}") + exec "${cmdline[@]}" & + disown + echo $! > "$pid_file" + exit 0 + fi + + cmdline+=("${args[@]}") + exec "${cmdline[@]}" + exit 1 +} + +# check self-install +if [ "$0" = "bash" ] || [[ "$0" =~ .*/bash ]]; then + if [ -d nextflow ]; then + echo 'Please note:' + echo "- The install procedure needs to create a file named 'nextflow' in this folder, but a directory with this name already exists." + echo "- Please renamed/delete that directory, or execute the Nextflow install procedure in another folder." + echo '' + exit 1 + fi + install "$PWD/nextflow" install + exit 0 +fi + +# clean up env +# see https://github.com/nextflow-io/nextflow/issues/1716 +unset JAVA_TOOL_OPTIONS + +# parse the command line +bg='' +declare -a jvmopts=() +declare -a args=("$@") +declare -a commands=(clone config drop help history info ls pull run view node console kuberun) +# $NXF_CLI_OPTS allow to pass arbitrary cli opts via the environment +# note: do not wrap with quotes because the variable can be used to specify more than on option separate by blanks +[ "$NXF_CLI_OPTS" ] && args+=($NXF_CLI_OPTS) + +cmd='' +while [[ $# != 0 ]]; do + case $1 in + -D*) + if [[ ! "$cmd" ]]; then + jvmopts+=("$1") + fi + ;; + -bg) + bg=1 + ;; + -remote-debug) + echo_yellow "Enabling script debugging - continue the execution launching the remote VM debugger in your favourite IDE using port $NXF_REMOTE_DEBUG_PORT" + remote_debug=1 + ;; + -download) + if [[ ! "$cmd" ]]; then + rm -rf "$NXF_DIST/$NXF_VER" || exit $? + bash "$0" -version || exit $? + exit 0 + fi + ;; + -self-update|self-update) + if [[ ! "$cmd" ]]; then + [[ -z $NXF_EDGE && $NXF_VER = *-edge ]] && NXF_EDGE=1 + unset NXF_VER + install "$0" update + exit 0 + fi + ;; + *) + [[ $1 && $1 != -* && ! "$cmd" && ${commands[*]} =~ $1 ]] && cmd=$1 + ;; + esac + shift +done + +CAPSULE_LOG=${CAPSULE_LOG:=''} +CAPSULE_RESET=${CAPSULE_RESET:=''} +CAPSULE_CACHE_DIR=${CAPSULE_CACHE_DIR:="$NXF_HOME/capsule"} + +NXF_PACK=one +NXF_MODE=${NXF_MODE:-''} +NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar} +NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR} +NXF_PATH=$(dirname "$NXF_BIN") +NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR} +NXF_GRAB=${NXF_GRAB:-''} +NXF_CLASSPATH=${NXF_CLASSPATH:-''} +NXF_HOST=${HOSTNAME:-localhost} +[[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST} + +# Determine the path to this file +if [[ $NXF_PACK = all ]]; then + NXF_BIN=$(which "$0" 2>/dev/null) + [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" +fi + +# use nextflow custom java home path +if [[ "$NXF_JAVA_HOME" ]]; then + JAVA_HOME="$NXF_JAVA_HOME" + unset JAVA_CMD +fi +# Determine the Java command to use to start the JVM. +if [ ! -x "$JAVA_CMD" ] ; then + if [ -d "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVA_CMD="$JAVA_HOME/jre/sh/java" + else + JAVA_CMD="$JAVA_HOME/bin/java" + fi + elif [ -x /usr/libexec/java_home ]; then + JAVA_CMD="$(/usr/libexec/java_home -v 1.8+)/bin/java" + else + JAVA_CMD="$(which java)" || JAVA_CMD=java + fi +fi + +# Retrieve the java version from a NF local file +JAVA_KEY="$NXF_HOME/tmp/ver/$(resolve_link "$JAVA_CMD" | sed 's@/@.@g')" +if [ -f "$JAVA_KEY" ]; then + JAVA_VER="$(cat "$JAVA_KEY")" +else + JAVA_VER="$("$JAVA_CMD" $NXF_OPTS -version 2>&1)" + if [ $? -ne 0 ]; then + echo_red "${JAVA_VER:-Failed to launch the Java virtual machine}" + echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n NXF_OPTS: $NXF_OPTS\n" + exit 1 + fi + JAVA_VER=$(echo "$JAVA_VER" | awk '/version/ {gsub(/"/, "", $3); print $3}') + # check NF version + if [[ ! $NXF_VER =~ ([0-9]+)\.([0-9]+)\.([0-9].*) ]]; then + echo_red "Not a valid Nextflow version: $NXF_VER" + exit 1 + fi + major=${BASH_REMATCH[1]} + minor=${BASH_REMATCH[2]} + # legacy version - Java 7/8 only + if [ $major -eq 0 ] && [ $minor -lt 26 ]; then + version_check="^(1.7|1.8)" + version_message="Java 7 or 8" + else + version_check="^(1.8|9|10|11|12|13|14|15|16|17|18|19|20|21)" + version_message="Java 8 or later (up to 21)" + fi + if [[ ! $JAVA_VER =~ $version_check ]]; then + echo_red "ERROR: Cannot find Java or it's a wrong version -- please make sure that $version_message is installed" + if [[ "$NXF_JAVA_HOME" ]]; then + echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n NXF_JAVA_HOME: $NXF_JAVA_HOME\n" + else + echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n JAVA_HOME: $JAVA_HOME\n" + fi + exit 1 + fi + if [[ ! $JAVA_VER =~ ^(11|12|13|14|15|16|17|18|19|20|21) ]]; then + echo_yellow "NOTE: Nextflow is not tested with Java $JAVA_VER -- It's recommended the use of version 11 up to 21\n" + fi + mkdir -p "$(dirname "$JAVA_KEY")" + [[ -f $JAVA_VER ]] && echo $JAVA_VER > "$JAVA_KEY" +fi + +# Verify nextflow jar is available +if [ ! -f "$NXF_BIN" ]; then + [ -f "$NXF_PATH" ] && rm "$NXF_PATH" + mkdir -p "$NXF_PATH" || exit $? + tmpfile=$(make_temp) + get "$NXF_URL" "$tmpfile" "$NXF_BIN" + mv "$tmpfile" "$NXF_BIN" +fi + +COLUMNS=${COLUMNS:-`tty -s && tput cols 2>/dev/null || true`} +declare -a JAVA_OPTS=() +JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcapsule.trampoline -Dcapsule.java.cmd="$JAVA_CMD" -Dcom.sun.security.enableAIAcaIssuers=true) +if [[ $cmd == console ]]; then bg=1; +else JAVA_OPTS+=(-Djava.awt.headless=true) +fi + +[[ "$JAVA_VER" =~ ^(21) ]] && [[ ! "$NXF_ENABLE_VIRTUAL_THREADS" ]] && NXF_ENABLE_VIRTUAL_THREADS=true +[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") +[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) +[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) +[[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1) +[[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS) +[[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH +[[ "$NXF_GRAB" ]] && export NXF_GRAB +[[ "$COLUMNS" ]] && export COLUMNS +[[ "$NXF_TEMP" ]] && JAVA_OPTS+=(-Djava.io.tmpdir="$NXF_TEMP") +[[ "${jvmopts[@]}" ]] && JAVA_OPTS+=("${jvmopts[@]}") +export JAVA_CMD +export CAPSULE_CACHE_DIR +export NXF_PLUGINS_DIR +export NXF_PLUGINS_MODE +export NXF_PLUGINS_DEFAULT +export NXF_PACK +export NXF_ENABLE_VIRTUAL_THREADS + +# lookup the a `md5` command +if hash md5sum 2>/dev/null; then MD5=md5sum; +elif hash gmd5sum 2>/dev/null; then MD5=gmd5sum; +elif hash md5 2>/dev/null; then MD5=md5; +else MD5='' +fi + +# when no md5 command is available fallback on default execution +if [ ! "$MD5" ] || [ "$CAPSULE_RESET" ]; then + launcher=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) + launch_nextflow + exit 1 +fi + +# creates a md5 unique for the given variables +env_md5() { +cat </dev/null; then + STR='' + for x in "${launcher[@]}"; do + [[ "$x" != "\"-Duser.dir=$PWD\"" ]] && [[ ! "$x" == *"-agentlib:jdwp"* ]] && STR+=$(printf '%q ' "$x") + done + printf "$STR">"$LAUNCH_FILE" + else + echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?" + fi + +fi + +# check for latest version +check_latest +# finally run it +launch_nextflow diff --git a/.latch/config b/.latch/config new file mode 100644 index 00000000..a8e1f41d --- /dev/null +++ b/.latch/config @@ -0,0 +1 @@ +{"latch_version": "2.46.6", "base_image": "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main", "date": "2024-06-17T18:20:03.014880"} \ No newline at end of file diff --git a/.latch/nextflow_version b/.latch/nextflow_version new file mode 100644 index 00000000..c1757a8d --- /dev/null +++ b/.latch/nextflow_version @@ -0,0 +1 @@ +v1.0.5 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3fba858e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# DO NOT CHANGE +from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main + +workdir /tmp/docker-build/work/ + +shell [ \ + "/usr/bin/env", "bash", \ + "-o", "errexit", \ + "-o", "pipefail", \ + "-o", "nounset", \ + "-o", "verbose", \ + "-o", "errtrace", \ + "-O", "inherit_errexit", \ + "-O", "shift_verbose", \ + "-c" \ +] +env TZ='Etc/UTC' +env LANG='en_US.UTF-8' + +arg DEBIAN_FRONTEND=noninteractive + +# Latch SDK +# DO NOT REMOVE +run pip install latch==2.46.6 +run mkdir /opt/latch +run apt-get update && apt-get install -y default-jre-headless + + +# Copy workflow data (use .dockerignore to skip files) + +copy . /root/ + +# Latch nextflow workflow entrypoint +# DO NOT CHANGE + +run ln -s /root/.latch/bin/nextflow /root/nextflow +run ln -s /root/.latch/.nextflow /root/.nextflow + + +# Latch workflow registration metadata +# DO NOT CHANGE +arg tag +# DO NOT CHANGE +env FLYTE_INTERNAL_IMAGE $tag + +workdir /root diff --git a/latch.config b/latch.config new file mode 100644 index 00000000..b8e37106 --- /dev/null +++ b/latch.config @@ -0,0 +1,9 @@ +process { + executor = 'k8s' +} + +aws { + client { + anonymous = true + } +} diff --git a/latch_metadata/__init__.py b/latch_metadata/__init__.py new file mode 100644 index 00000000..d2738853 --- /dev/null +++ b/latch_metadata/__init__.py @@ -0,0 +1,23 @@ + +from latch.types.metadata import ( + NextflowMetadata, + LatchAuthor, + NextflowRuntimeResources +) +from latch.types.directory import LatchDir + +from .parameters import generated_parameters + +NextflowMetadata( + display_name='nf-core/cutandrun', + author=LatchAuthor( + name="Your Name", + ), + parameters=generated_parameters, + runtime_resources=NextflowRuntimeResources( + cpus=4, + memory=8, + storage_gib=100, + ), + log_dir=LatchDir("latch:///your_log_dir"), +) diff --git a/latch_metadata/parameters.py b/latch_metadata/parameters.py new file mode 100644 index 00000000..da8993ed --- /dev/null +++ b/latch_metadata/parameters.py @@ -0,0 +1,480 @@ + +from dataclasses import dataclass +import typing +import typing_extensions + +from flytekit.core.annotation import FlyteAnnotation + +from latch.types.metadata import NextflowParameter +from latch.types.file import LatchFile +from latch.types.directory import LatchDir, LatchOutputDir + +# Import these into your `__init__.py` file: +# +# from .parameters import generated_parameters + +generated_parameters = { + 'input': NextflowParameter( + type=LatchFile, + default=None, + section_title='Input/output options', + description='Path to comma-separated file containing information about the samples in the experiment.', + ), + 'outdir': NextflowParameter( + type=typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], + default=None, + section_title=None, + description='The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.', + ), + 'multiqc_title': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='MultiQC report title. Printed as page header, used for filename if not otherwise specified.', + ), + 'save_reference': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save genome reference data to the output directory', + ), + 'save_merged_fastq': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save any technical replicate FASTQ files that were merged to the output directory', + ), + 'save_trimmed': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save trimmed FASTQ files to the output directory', + ), + 'save_spikein_aligned': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save BAM files aligned to the spike-in genome to the output directory', + ), + 'save_unaligned': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save unaligned sequences to the output directory', + ), + 'save_align_intermed': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Save alignment intermediates to the output directory (WARNING: can be very large)', + ), + 'email': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Email address for completion summary.', + ), + 'dump_scale_factors': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Output calculated scale factors from pipeline', + ), + 'genome': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title='Reference data options', + description='Name of iGenomes reference.', + ), + 'bowtie2': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to bowtie2 index', + ), + 'gtf': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to GTF annotation file', + ), + 'gene_bed': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to gene BED file', + ), + 'blacklist': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to genome blacklist', + ), + 'spikein_genome': NextflowParameter( + type=typing.Optional[str], + default='K12-MG1655', + section_title=None, + description='Name of the iGenome reference for the spike-in genome, defaulting to E. coli K12, for yeast set to R64-1-1, for fruit fly BDGP6', + ), + 'spikein_bowtie2': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to spike-in bowtie2 index', + ), + 'spikein_fasta': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Path to spike-in fasta', + ), + 'fasta': NextflowParameter( + type=typing.Optional[LatchFile], + default=None, + section_title=None, + description='Path to FASTA genome file.', + ), + 'only_input': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title='Flow switching options', + description='Run pipeline up to input checking', + ), + 'only_genome': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Run pipeline up to reference preparation', + ), + 'only_preqc': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Run pipeline up to pre-alignment', + ), + 'only_alignment': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Run pipeline up to alignment', + ), + 'only_filtering': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Run pipeline up to q-filtering', + ), + 'only_peak_calling': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Run pipeline up to peak calling', + ), + 'skip_fastqc': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips fastqc reporting', + ), + 'skip_trimming': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips trimming', + ), + 'skip_removeduplicates': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips de-duplication', + ), + 'skip_reporting': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips reporting', + ), + 'skip_preseq': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips preseq reporting', + ), + 'skip_igv': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips igv session generation', + ), + 'skip_dt_qc': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips deeptools QC repoting', + ), + 'skip_heatmaps': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips deeptools heatmap generation', + ), + 'skip_peak_qc': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips peak QC reporting', + ), + 'skip_multiqc': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Skips multiqc', + ), + 'clip_r1': NextflowParameter( + type=typing.Optional[int], + default=0, + section_title='Trimming Options', + description="Instructs Trim Galore to remove bp from the 5' end of read 1 (or single-end reads).", + ), + 'clip_r2': NextflowParameter( + type=typing.Optional[int], + default=0, + section_title=None, + description="Instructs Trim Galore to remove bp from the 5' end of read 2 (paired-end reads only).", + ), + 'three_prime_clip_r1': NextflowParameter( + type=typing.Optional[int], + default=0, + section_title=None, + description="Instructs Trim Galore to remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed.", + ), + 'three_prime_clip_r2': NextflowParameter( + type=typing.Optional[int], + default=0, + section_title=None, + description="Instructs Trim Galore to remove bp from the 3' end of read 2 AFTER adapter/quality trimming has been performed.", + ), + 'trim_nextseq': NextflowParameter( + type=typing.Optional[int], + default=0, + section_title=None, + description='Instructs Trim Galore to apply the --nextseq=X option, to trim based on quality after removing poly-G tails.', + ), + 'minimum_alignment_q_score': NextflowParameter( + type=typing.Optional[int], + default=20, + section_title='Pipeline Options', + description='Filter reads below a q-score threshold', + ), + 'remove_mitochondrial_reads': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='Filter mitochondrial reads', + ), + 'mito_name': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Name of mitochondrial reads in reference genome. Only necessary when using a custom (non-igenomes) reference genome.', + ), + 'dedup_target_reads': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description='De-duplicate target reads AND control reads (default is control only)', + ), + 'remove_linear_duplicates': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title=None, + description="De-duplicate reads based on read 1 5' start position. Relevant for assays using linear amplification with tagmentation (default is false).", + ), + 'end_to_end': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Use --end-to-end mode of Bowtie2 during alignment', + ), + 'normalisation_mode': NextflowParameter( + type=typing.Optional[str], + default='Spikein', + section_title=None, + description='Sets the target read normalisation mode. Options are: ["Spikein", "RPKM", "CPM", "BPM", "None" ]', + ), + 'normalisation_binsize': NextflowParameter( + type=typing.Optional[int], + default=50, + section_title=None, + description='If normsalisation option is one of "RPKM", "CPM", "BPM" - then the binsize that the reads count is calculated on is used.', + ), + 'peakcaller': NextflowParameter( + type=typing.Optional[str], + default='seacr', + section_title=None, + description='Selects the peak caller for the pipeline. Options are: [seacr, macs2]. More than one peak caller can be chosen and the order specifies which is a primary peak called (the first) that will be used downstream. Any secondary peak callers will be run and outputed to the results folder.', + ), + 'use_control': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Specifies whether to use a control to normalise peak calls against (e.g. IgG)', + ), + 'extend_fragments': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Specifies whether to extend paired-end fragments between the read mates when calculating coveage tracks', + ), + 'igg_scale_factor': NextflowParameter( + type=typing.Optional[float], + default=0.5, + section_title=None, + description='Specifies whether the background control is scaled prior to being used to normalise peaks.', + ), + 'seacr_peak_threshold': NextflowParameter( + type=typing.Optional[float], + default=0.05, + section_title=None, + description='SEACR specifies returns the top n fraction (between 0 and 1) of peaks based on total signal within peaks. This is only used if there are no controls included with the samples and if `--use_control` is `false`', + ), + 'seacr_norm': NextflowParameter( + type=typing.Optional[str], + default='non', + section_title=None, + description='SEACR normalization. ', + ), + 'seacr_stringent': NextflowParameter( + type=typing.Optional[str], + default='stringent', + section_title=None, + description='SEACR stringency.', + ), + 'macs2_qvalue': NextflowParameter( + type=typing.Optional[float], + default=0.01, + section_title=None, + description='Q-value threshold for MACS2 peak caller.', + ), + 'macs2_pvalue': NextflowParameter( + type=typing.Optional[float], + default=None, + section_title=None, + description='P-value threshold for macs2 peak caller. If set it will overide the qvalue.', + ), + 'macs_gsize': NextflowParameter( + type=typing.Optional[float], + default=2700000000, + section_title=None, + description='parameter required by MACS2. If using an iGenomes reference these have been provided when `--genome` is set as *GRCh37*, *GRCh38*, *GRCm38*, *WBcel235*, *BDGP6*, *R64-1-1*, *EF2*, *hg38*, *hg19* and *mm10*. Otherwise the gsize will default to GRCh38.', + ), + 'macs2_narrow_peak': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Determines whether MACS2 broad or narrow peak mode is used for the peak caller', + ), + 'macs2_broad_cutoff': NextflowParameter( + type=typing.Optional[float], + default=0.1, + section_title=None, + description='MACS2 broad cutoff parameter', + ), + 'consensus_peak_mode': NextflowParameter( + type=typing.Optional[str], + default='group', + section_title=None, + description='Specifies what samples to group together for consensus peaks. Options are [group, all]', + ), + 'replicate_threshold': NextflowParameter( + type=typing.Optional[float], + default=1, + section_title=None, + description='Minimum number of overlapping replicates needed for a consensus peak', + ), + 'igv_show_gene_names': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Show gene names instead of symbols in IGV browser sessions', + ), + 'dt_qc_bam_binsize': NextflowParameter( + type=typing.Optional[int], + default=500, + section_title='Reporting Options', + description='Deeptools multiBamSummary bam bin size', + ), + 'dt_qc_corr_method': NextflowParameter( + type=typing.Optional[str], + default='pearson', + section_title=None, + description='Deeptools Correlation Plot statistical calculation method', + ), + 'dt_heatmap_gene_beforelen': NextflowParameter( + type=typing.Optional[int], + default=3000, + section_title=None, + description='Deeptools heatmap gene plot before length (bases)', + ), + 'dt_heatmap_gene_bodylen': NextflowParameter( + type=typing.Optional[int], + default=5000, + section_title=None, + description='Deeptools heatmap gene plot body length (bases)', + ), + 'dt_heatmap_gene_afterlen': NextflowParameter( + type=typing.Optional[int], + default=3000, + section_title=None, + description='Deeptools heatmap gene plot after length (bases)', + ), + 'dt_heatmap_peak_beforelen': NextflowParameter( + type=typing.Optional[int], + default=3000, + section_title=None, + description='Deeptools heatmap peak plot before length (bases)', + ), + 'dt_heatmap_peak_afterlen': NextflowParameter( + type=typing.Optional[int], + default=3000, + section_title=None, + description='Deeptools heatmap peak plot after length (bases)', + ), + 'dt_calc_all_matrix': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Flag for whether to generate a heatmap for all samples together', + ), + 'min_frip_overlap': NextflowParameter( + type=typing.Optional[float], + default=0.2, + section_title=None, + description='Minimum fragment overlap for FriP score', + ), + 'min_peak_overlap': NextflowParameter( + type=typing.Optional[float], + default=0.2, + section_title=None, + description='Minimum peak overlap for peak reproducibility plot', + ), + 'igv_sort_by_groups': NextflowParameter( + type=typing.Optional[bool], + default=True, + section_title=None, + description='Sort the IGV output tracks by group', + ), + 'singularity_pull_docker_container': NextflowParameter( + type=typing.Optional[bool], + default=None, + section_title='Generic options', + description='Pull Docker container.', + ), + 'multiqc_methods_description': NextflowParameter( + type=typing.Optional[str], + default=None, + section_title=None, + description='Custom MultiQC yaml file containing HTML including a methods description.', + ), +} + diff --git a/version b/version new file mode 100644 index 00000000..6e8bf73a --- /dev/null +++ b/version @@ -0,0 +1 @@ +0.1.0 diff --git a/wf/entrypoint.py b/wf/entrypoint.py new file mode 100644 index 00000000..26524c50 --- /dev/null +++ b/wf/entrypoint.py @@ -0,0 +1,212 @@ +from dataclasses import dataclass +from enum import Enum +import os +import subprocess +import requests +import shutil +from pathlib import Path +import typing +import typing_extensions + +from latch.resources.workflow import workflow +from latch.resources.tasks import nextflow_runtime_task, custom_task +from latch.types.file import LatchFile +from latch.types.directory import LatchDir, LatchOutputDir +from latch.ldata.path import LPath +from latch_cli.nextflow.workflow import get_flag +from latch_cli.nextflow.utils import _get_execution_name +from latch_cli.utils import urljoins +from latch.types import metadata +from flytekit.core.annotation import FlyteAnnotation + +from latch_cli.services.register.utils import import_module_by_path + +meta = Path("latch_metadata") / "__init__.py" +import_module_by_path(meta) +import latch_metadata + +@custom_task(cpu=0.25, memory=0.5, storage_gib=1) +def initialize() -> str: + token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID") + if token is None: + raise RuntimeError("failed to get execution token") + + headers = {"Authorization": f"Latch-Execution-Token {token}"} + + print("Provisioning shared storage volume... ", end="") + resp = requests.post( + "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage", + headers=headers, + json={ + "storage_gib": 100, + } + ) + resp.raise_for_status() + print("Done.") + + return resp.json()["name"] + + + + + + +@nextflow_runtime_task(cpu=4, memory=8, storage_gib=100) +def nextflow_runtime(pvc_name: str, input: LatchFile, outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], multiqc_title: typing.Optional[str], save_reference: typing.Optional[bool], save_merged_fastq: typing.Optional[bool], save_trimmed: typing.Optional[bool], save_spikein_aligned: typing.Optional[bool], save_unaligned: typing.Optional[bool], save_align_intermed: typing.Optional[bool], email: typing.Optional[str], dump_scale_factors: typing.Optional[bool], genome: typing.Optional[str], bowtie2: typing.Optional[str], gtf: typing.Optional[str], gene_bed: typing.Optional[str], blacklist: typing.Optional[str], spikein_bowtie2: typing.Optional[str], spikein_fasta: typing.Optional[str], fasta: typing.Optional[LatchFile], only_input: typing.Optional[bool], only_genome: typing.Optional[bool], only_preqc: typing.Optional[bool], only_alignment: typing.Optional[bool], only_filtering: typing.Optional[bool], only_peak_calling: typing.Optional[bool], skip_fastqc: typing.Optional[bool], skip_trimming: typing.Optional[bool], skip_removeduplicates: typing.Optional[bool], skip_reporting: typing.Optional[bool], skip_preseq: typing.Optional[bool], skip_igv: typing.Optional[bool], skip_dt_qc: typing.Optional[bool], skip_heatmaps: typing.Optional[bool], skip_peak_qc: typing.Optional[bool], skip_multiqc: typing.Optional[bool], remove_mitochondrial_reads: typing.Optional[bool], mito_name: typing.Optional[str], dedup_target_reads: typing.Optional[bool], remove_linear_duplicates: typing.Optional[bool], macs2_pvalue: typing.Optional[float], singularity_pull_docker_container: typing.Optional[bool], multiqc_methods_description: typing.Optional[str], spikein_genome: typing.Optional[str], clip_r1: typing.Optional[int], clip_r2: typing.Optional[int], three_prime_clip_r1: typing.Optional[int], three_prime_clip_r2: typing.Optional[int], trim_nextseq: typing.Optional[int], minimum_alignment_q_score: typing.Optional[int], end_to_end: typing.Optional[bool], normalisation_mode: typing.Optional[str], normalisation_binsize: typing.Optional[int], peakcaller: typing.Optional[str], use_control: typing.Optional[bool], extend_fragments: typing.Optional[bool], igg_scale_factor: typing.Optional[float], seacr_peak_threshold: typing.Optional[float], seacr_norm: typing.Optional[str], seacr_stringent: typing.Optional[str], macs2_qvalue: typing.Optional[float], macs_gsize: typing.Optional[float], macs2_narrow_peak: typing.Optional[bool], macs2_broad_cutoff: typing.Optional[float], consensus_peak_mode: typing.Optional[str], replicate_threshold: typing.Optional[float], igv_show_gene_names: typing.Optional[bool], dt_qc_bam_binsize: typing.Optional[int], dt_qc_corr_method: typing.Optional[str], dt_heatmap_gene_beforelen: typing.Optional[int], dt_heatmap_gene_bodylen: typing.Optional[int], dt_heatmap_gene_afterlen: typing.Optional[int], dt_heatmap_peak_beforelen: typing.Optional[int], dt_heatmap_peak_afterlen: typing.Optional[int], dt_calc_all_matrix: typing.Optional[bool], min_frip_overlap: typing.Optional[float], min_peak_overlap: typing.Optional[float], igv_sort_by_groups: typing.Optional[bool]) -> None: + try: + shared_dir = Path("/nf-workdir") + + + + ignore_list = [ + "latch", + ".latch", + "nextflow", + ".nextflow", + "work", + "results", + "miniconda", + "anaconda3", + "mambaforge", + ] + + shutil.copytree( + Path("/root"), + shared_dir, + ignore=lambda src, names: ignore_list, + ignore_dangling_symlinks=True, + dirs_exist_ok=True, + ) + + cmd = [ + "/root/nextflow", + "run", + str(shared_dir / "main.nf"), + "-work-dir", + str(shared_dir), + "-profile", + "docker", + "-c", + "latch.config", + *get_flag('input', input), + *get_flag('outdir', outdir), + *get_flag('multiqc_title', multiqc_title), + *get_flag('save_reference', save_reference), + *get_flag('save_merged_fastq', save_merged_fastq), + *get_flag('save_trimmed', save_trimmed), + *get_flag('save_spikein_aligned', save_spikein_aligned), + *get_flag('save_unaligned', save_unaligned), + *get_flag('save_align_intermed', save_align_intermed), + *get_flag('email', email), + *get_flag('dump_scale_factors', dump_scale_factors), + *get_flag('genome', genome), + *get_flag('bowtie2', bowtie2), + *get_flag('gtf', gtf), + *get_flag('gene_bed', gene_bed), + *get_flag('blacklist', blacklist), + *get_flag('spikein_genome', spikein_genome), + *get_flag('spikein_bowtie2', spikein_bowtie2), + *get_flag('spikein_fasta', spikein_fasta), + *get_flag('fasta', fasta), + *get_flag('only_input', only_input), + *get_flag('only_genome', only_genome), + *get_flag('only_preqc', only_preqc), + *get_flag('only_alignment', only_alignment), + *get_flag('only_filtering', only_filtering), + *get_flag('only_peak_calling', only_peak_calling), + *get_flag('skip_fastqc', skip_fastqc), + *get_flag('skip_trimming', skip_trimming), + *get_flag('skip_removeduplicates', skip_removeduplicates), + *get_flag('skip_reporting', skip_reporting), + *get_flag('skip_preseq', skip_preseq), + *get_flag('skip_igv', skip_igv), + *get_flag('skip_dt_qc', skip_dt_qc), + *get_flag('skip_heatmaps', skip_heatmaps), + *get_flag('skip_peak_qc', skip_peak_qc), + *get_flag('skip_multiqc', skip_multiqc), + *get_flag('clip_r1', clip_r1), + *get_flag('clip_r2', clip_r2), + *get_flag('three_prime_clip_r1', three_prime_clip_r1), + *get_flag('three_prime_clip_r2', three_prime_clip_r2), + *get_flag('trim_nextseq', trim_nextseq), + *get_flag('minimum_alignment_q_score', minimum_alignment_q_score), + *get_flag('remove_mitochondrial_reads', remove_mitochondrial_reads), + *get_flag('mito_name', mito_name), + *get_flag('dedup_target_reads', dedup_target_reads), + *get_flag('remove_linear_duplicates', remove_linear_duplicates), + *get_flag('end_to_end', end_to_end), + *get_flag('normalisation_mode', normalisation_mode), + *get_flag('normalisation_binsize', normalisation_binsize), + *get_flag('peakcaller', peakcaller), + *get_flag('use_control', use_control), + *get_flag('extend_fragments', extend_fragments), + *get_flag('igg_scale_factor', igg_scale_factor), + *get_flag('seacr_peak_threshold', seacr_peak_threshold), + *get_flag('seacr_norm', seacr_norm), + *get_flag('seacr_stringent', seacr_stringent), + *get_flag('macs2_qvalue', macs2_qvalue), + *get_flag('macs2_pvalue', macs2_pvalue), + *get_flag('macs_gsize', macs_gsize), + *get_flag('macs2_narrow_peak', macs2_narrow_peak), + *get_flag('macs2_broad_cutoff', macs2_broad_cutoff), + *get_flag('consensus_peak_mode', consensus_peak_mode), + *get_flag('replicate_threshold', replicate_threshold), + *get_flag('igv_show_gene_names', igv_show_gene_names), + *get_flag('dt_qc_bam_binsize', dt_qc_bam_binsize), + *get_flag('dt_qc_corr_method', dt_qc_corr_method), + *get_flag('dt_heatmap_gene_beforelen', dt_heatmap_gene_beforelen), + *get_flag('dt_heatmap_gene_bodylen', dt_heatmap_gene_bodylen), + *get_flag('dt_heatmap_gene_afterlen', dt_heatmap_gene_afterlen), + *get_flag('dt_heatmap_peak_beforelen', dt_heatmap_peak_beforelen), + *get_flag('dt_heatmap_peak_afterlen', dt_heatmap_peak_afterlen), + *get_flag('dt_calc_all_matrix', dt_calc_all_matrix), + *get_flag('min_frip_overlap', min_frip_overlap), + *get_flag('min_peak_overlap', min_peak_overlap), + *get_flag('igv_sort_by_groups', igv_sort_by_groups), + *get_flag('singularity_pull_docker_container', singularity_pull_docker_container), + *get_flag('multiqc_methods_description', multiqc_methods_description) + ] + + print("Launching Nextflow Runtime") + print(' '.join(cmd)) + print(flush=True) + + env = { + **os.environ, + "NXF_HOME": "/root/.nextflow", + "NXF_OPTS": "-Xms2048M -Xmx8G -XX:ActiveProcessorCount=4", + "K8S_STORAGE_CLAIM_NAME": pvc_name, + "NXF_DISABLE_CHECK_LATEST": "true", + } + subprocess.run( + cmd, + env=env, + check=True, + cwd=str(shared_dir), + ) + finally: + print() + + nextflow_log = shared_dir / ".nextflow.log" + if nextflow_log.exists(): + name = _get_execution_name() + if name is None: + print("Skipping logs upload, failed to get execution name") + else: + remote = LPath(urljoins("latch:///your_log_dir/nf_nf_core_cutandrun", name, "nextflow.log")) + print(f"Uploading .nextflow.log to {remote.path}") + remote.upload_from(nextflow_log) + + + +@workflow(metadata._nextflow_metadata) +def nf_nf_core_cutandrun(input: LatchFile, outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], multiqc_title: typing.Optional[str], save_reference: typing.Optional[bool], save_merged_fastq: typing.Optional[bool], save_trimmed: typing.Optional[bool], save_spikein_aligned: typing.Optional[bool], save_unaligned: typing.Optional[bool], save_align_intermed: typing.Optional[bool], email: typing.Optional[str], dump_scale_factors: typing.Optional[bool], genome: typing.Optional[str], bowtie2: typing.Optional[str], gtf: typing.Optional[str], gene_bed: typing.Optional[str], blacklist: typing.Optional[str], spikein_bowtie2: typing.Optional[str], spikein_fasta: typing.Optional[str], fasta: typing.Optional[LatchFile], only_input: typing.Optional[bool], only_genome: typing.Optional[bool], only_preqc: typing.Optional[bool], only_alignment: typing.Optional[bool], only_filtering: typing.Optional[bool], only_peak_calling: typing.Optional[bool], skip_fastqc: typing.Optional[bool], skip_trimming: typing.Optional[bool], skip_removeduplicates: typing.Optional[bool], skip_reporting: typing.Optional[bool], skip_preseq: typing.Optional[bool], skip_igv: typing.Optional[bool], skip_dt_qc: typing.Optional[bool], skip_heatmaps: typing.Optional[bool], skip_peak_qc: typing.Optional[bool], skip_multiqc: typing.Optional[bool], remove_mitochondrial_reads: typing.Optional[bool], mito_name: typing.Optional[str], dedup_target_reads: typing.Optional[bool], remove_linear_duplicates: typing.Optional[bool], macs2_pvalue: typing.Optional[float], singularity_pull_docker_container: typing.Optional[bool], multiqc_methods_description: typing.Optional[str], spikein_genome: typing.Optional[str] = 'K12-MG1655', clip_r1: typing.Optional[int] = 0, clip_r2: typing.Optional[int] = 0, three_prime_clip_r1: typing.Optional[int] = 0, three_prime_clip_r2: typing.Optional[int] = 0, trim_nextseq: typing.Optional[int] = 0, minimum_alignment_q_score: typing.Optional[int] = 20, end_to_end: typing.Optional[bool] = True, normalisation_mode: typing.Optional[str] = 'Spikein', normalisation_binsize: typing.Optional[int] = 50, peakcaller: typing.Optional[str] = 'seacr', use_control: typing.Optional[bool] = True, extend_fragments: typing.Optional[bool] = True, igg_scale_factor: typing.Optional[float] = 0.5, seacr_peak_threshold: typing.Optional[float] = 0.05, seacr_norm: typing.Optional[str] = 'non', seacr_stringent: typing.Optional[str] = 'stringent', macs2_qvalue: typing.Optional[float] = 0.01, macs_gsize: typing.Optional[float] = 2700000000, macs2_narrow_peak: typing.Optional[bool] = True, macs2_broad_cutoff: typing.Optional[float] = 0.1, consensus_peak_mode: typing.Optional[str] = 'group', replicate_threshold: typing.Optional[float] = 1, igv_show_gene_names: typing.Optional[bool] = True, dt_qc_bam_binsize: typing.Optional[int] = 500, dt_qc_corr_method: typing.Optional[str] = 'pearson', dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, dt_calc_all_matrix: typing.Optional[bool] = True, min_frip_overlap: typing.Optional[float] = 0.2, min_peak_overlap: typing.Optional[float] = 0.2, igv_sort_by_groups: typing.Optional[bool] = True) -> None: + """ + nf-core/cutandrun + + Sample Description + """ + + pvc_name: str = initialize() + nextflow_runtime(pvc_name=pvc_name, input=input, outdir=outdir, multiqc_title=multiqc_title, save_reference=save_reference, save_merged_fastq=save_merged_fastq, save_trimmed=save_trimmed, save_spikein_aligned=save_spikein_aligned, save_unaligned=save_unaligned, save_align_intermed=save_align_intermed, email=email, dump_scale_factors=dump_scale_factors, genome=genome, bowtie2=bowtie2, gtf=gtf, gene_bed=gene_bed, blacklist=blacklist, spikein_genome=spikein_genome, spikein_bowtie2=spikein_bowtie2, spikein_fasta=spikein_fasta, fasta=fasta, only_input=only_input, only_genome=only_genome, only_preqc=only_preqc, only_alignment=only_alignment, only_filtering=only_filtering, only_peak_calling=only_peak_calling, skip_fastqc=skip_fastqc, skip_trimming=skip_trimming, skip_removeduplicates=skip_removeduplicates, skip_reporting=skip_reporting, skip_preseq=skip_preseq, skip_igv=skip_igv, skip_dt_qc=skip_dt_qc, skip_heatmaps=skip_heatmaps, skip_peak_qc=skip_peak_qc, skip_multiqc=skip_multiqc, clip_r1=clip_r1, clip_r2=clip_r2, three_prime_clip_r1=three_prime_clip_r1, three_prime_clip_r2=three_prime_clip_r2, trim_nextseq=trim_nextseq, minimum_alignment_q_score=minimum_alignment_q_score, remove_mitochondrial_reads=remove_mitochondrial_reads, mito_name=mito_name, dedup_target_reads=dedup_target_reads, remove_linear_duplicates=remove_linear_duplicates, end_to_end=end_to_end, normalisation_mode=normalisation_mode, normalisation_binsize=normalisation_binsize, peakcaller=peakcaller, use_control=use_control, extend_fragments=extend_fragments, igg_scale_factor=igg_scale_factor, seacr_peak_threshold=seacr_peak_threshold, seacr_norm=seacr_norm, seacr_stringent=seacr_stringent, macs2_qvalue=macs2_qvalue, macs2_pvalue=macs2_pvalue, macs_gsize=macs_gsize, macs2_narrow_peak=macs2_narrow_peak, macs2_broad_cutoff=macs2_broad_cutoff, consensus_peak_mode=consensus_peak_mode, replicate_threshold=replicate_threshold, igv_show_gene_names=igv_show_gene_names, dt_qc_bam_binsize=dt_qc_bam_binsize, dt_qc_corr_method=dt_qc_corr_method, dt_heatmap_gene_beforelen=dt_heatmap_gene_beforelen, dt_heatmap_gene_bodylen=dt_heatmap_gene_bodylen, dt_heatmap_gene_afterlen=dt_heatmap_gene_afterlen, dt_heatmap_peak_beforelen=dt_heatmap_peak_beforelen, dt_heatmap_peak_afterlen=dt_heatmap_peak_afterlen, dt_calc_all_matrix=dt_calc_all_matrix, min_frip_overlap=min_frip_overlap, min_peak_overlap=min_peak_overlap, igv_sort_by_groups=igv_sort_by_groups, singularity_pull_docker_container=singularity_pull_docker_container, multiqc_methods_description=multiqc_methods_description) + From 96a42f9d96daa80f291e838ca35ebd7273c8b5c6 Mon Sep 17 00:00:00 2001 From: Tahir D'Mello Date: Mon, 17 Jun 2024 20:36:48 -0700 Subject: [PATCH 2/5] float default fix --- .../docker-build-logs.txt | 109 +++++ .latch/workflow_name | 1 + wf/entrypoint.py | 435 +++++++++++++----- 3 files changed, 440 insertions(+), 105 deletions(-) create mode 100644 .latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt create mode 100644 .latch/workflow_name diff --git a/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt b/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt new file mode 100644 index 00000000..c5ff52dc --- /dev/null +++ b/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt @@ -0,0 +1,109 @@ +Step 1/15 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main + + + ---> f23c37efcd53 + +Step 2/15 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> dd16c2d4f334 + +Step 3/15 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> aaada0c60239 + +Step 4/15 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> d6869bc03deb + +Step 5/15 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> ae2f7ca65880 + +Step 6/15 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> f029b3f36a86 + +Step 7/15 : run pip install latch==2.46.6 + + + ---> Using cache + + ---> be1350f11537 + +Step 8/15 : run mkdir /opt/latch + + + ---> Using cache + + ---> e8dd4419614b + +Step 9/15 : run apt-get update && apt-get install -y default-jre-headless + + + ---> Using cache + + ---> 447cbd90cbc9 + +Step 10/15 : copy . /root/ + + + ---> a9468c7b7bff + +Step 11/15 : run ln -s /root/.latch/bin/nextflow /root/nextflow + + + ---> Running in d056c3d7bd82 + +ln -s /root/.latch/bin/nextflow /root/nextflow + + ---> 8bb36367c2d8 + +Step 12/15 : run ln -s /root/.latch/.nextflow /root/.nextflow + + + ---> Running in 1abcae1835aa + +ln -s /root/.latch/.nextflow /root/.nextflow + + ---> 50622da6d862 + +Step 13/15 : arg tag + + + ---> Running in ebadde0805f8 + + ---> c030684dc96f + +Step 14/15 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 22e9ae745493 + + ---> ce25eee14447 + +Step 15/15 : workdir /root + + + ---> Running in eec1c7ae0d2d + + ---> 2f082b780475 + +Successfully built 2f082b780475 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250 + diff --git a/.latch/workflow_name b/.latch/workflow_name new file mode 100644 index 00000000..3f929e9f --- /dev/null +++ b/.latch/workflow_name @@ -0,0 +1 @@ +wf.entrypoint.nf_nf_core_cutandrun \ No newline at end of file diff --git a/wf/entrypoint.py b/wf/entrypoint.py index 26524c50..98eae829 100644 --- a/wf/entrypoint.py +++ b/wf/entrypoint.py @@ -1,29 +1,26 @@ -from dataclasses import dataclass -from enum import Enum import os -import subprocess -import requests import shutil -from pathlib import Path +import subprocess import typing -import typing_extensions +from pathlib import Path +import requests +import typing_extensions +from flytekit.core.annotation import FlyteAnnotation +from latch.ldata.path import LPath +from latch.resources.tasks import custom_task, nextflow_runtime_task from latch.resources.workflow import workflow -from latch.resources.tasks import nextflow_runtime_task, custom_task +from latch.types import metadata +from latch.types.directory import LatchDir from latch.types.file import LatchFile -from latch.types.directory import LatchDir, LatchOutputDir -from latch.ldata.path import LPath -from latch_cli.nextflow.workflow import get_flag from latch_cli.nextflow.utils import _get_execution_name -from latch_cli.utils import urljoins -from latch.types import metadata -from flytekit.core.annotation import FlyteAnnotation - +from latch_cli.nextflow.workflow import get_flag from latch_cli.services.register.utils import import_module_by_path +from latch_cli.utils import urljoins meta = Path("latch_metadata") / "__init__.py" import_module_by_path(meta) -import latch_metadata + @custom_task(cpu=0.25, memory=0.5, storage_gib=1) def initialize() -> str: @@ -39,7 +36,7 @@ def initialize() -> str: headers=headers, json={ "storage_gib": 100, - } + }, ) resp.raise_for_status() print("Done.") @@ -47,17 +44,90 @@ def initialize() -> str: return resp.json()["name"] - - - - @nextflow_runtime_task(cpu=4, memory=8, storage_gib=100) -def nextflow_runtime(pvc_name: str, input: LatchFile, outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], multiqc_title: typing.Optional[str], save_reference: typing.Optional[bool], save_merged_fastq: typing.Optional[bool], save_trimmed: typing.Optional[bool], save_spikein_aligned: typing.Optional[bool], save_unaligned: typing.Optional[bool], save_align_intermed: typing.Optional[bool], email: typing.Optional[str], dump_scale_factors: typing.Optional[bool], genome: typing.Optional[str], bowtie2: typing.Optional[str], gtf: typing.Optional[str], gene_bed: typing.Optional[str], blacklist: typing.Optional[str], spikein_bowtie2: typing.Optional[str], spikein_fasta: typing.Optional[str], fasta: typing.Optional[LatchFile], only_input: typing.Optional[bool], only_genome: typing.Optional[bool], only_preqc: typing.Optional[bool], only_alignment: typing.Optional[bool], only_filtering: typing.Optional[bool], only_peak_calling: typing.Optional[bool], skip_fastqc: typing.Optional[bool], skip_trimming: typing.Optional[bool], skip_removeduplicates: typing.Optional[bool], skip_reporting: typing.Optional[bool], skip_preseq: typing.Optional[bool], skip_igv: typing.Optional[bool], skip_dt_qc: typing.Optional[bool], skip_heatmaps: typing.Optional[bool], skip_peak_qc: typing.Optional[bool], skip_multiqc: typing.Optional[bool], remove_mitochondrial_reads: typing.Optional[bool], mito_name: typing.Optional[str], dedup_target_reads: typing.Optional[bool], remove_linear_duplicates: typing.Optional[bool], macs2_pvalue: typing.Optional[float], singularity_pull_docker_container: typing.Optional[bool], multiqc_methods_description: typing.Optional[str], spikein_genome: typing.Optional[str], clip_r1: typing.Optional[int], clip_r2: typing.Optional[int], three_prime_clip_r1: typing.Optional[int], three_prime_clip_r2: typing.Optional[int], trim_nextseq: typing.Optional[int], minimum_alignment_q_score: typing.Optional[int], end_to_end: typing.Optional[bool], normalisation_mode: typing.Optional[str], normalisation_binsize: typing.Optional[int], peakcaller: typing.Optional[str], use_control: typing.Optional[bool], extend_fragments: typing.Optional[bool], igg_scale_factor: typing.Optional[float], seacr_peak_threshold: typing.Optional[float], seacr_norm: typing.Optional[str], seacr_stringent: typing.Optional[str], macs2_qvalue: typing.Optional[float], macs_gsize: typing.Optional[float], macs2_narrow_peak: typing.Optional[bool], macs2_broad_cutoff: typing.Optional[float], consensus_peak_mode: typing.Optional[str], replicate_threshold: typing.Optional[float], igv_show_gene_names: typing.Optional[bool], dt_qc_bam_binsize: typing.Optional[int], dt_qc_corr_method: typing.Optional[str], dt_heatmap_gene_beforelen: typing.Optional[int], dt_heatmap_gene_bodylen: typing.Optional[int], dt_heatmap_gene_afterlen: typing.Optional[int], dt_heatmap_peak_beforelen: typing.Optional[int], dt_heatmap_peak_afterlen: typing.Optional[int], dt_calc_all_matrix: typing.Optional[bool], min_frip_overlap: typing.Optional[float], min_peak_overlap: typing.Optional[float], igv_sort_by_groups: typing.Optional[bool]) -> None: +def nextflow_runtime( + pvc_name: str, + input: LatchFile, + outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], + multiqc_title: typing.Optional[str], + save_reference: typing.Optional[bool], + save_merged_fastq: typing.Optional[bool], + save_trimmed: typing.Optional[bool], + save_spikein_aligned: typing.Optional[bool], + save_unaligned: typing.Optional[bool], + save_align_intermed: typing.Optional[bool], + email: typing.Optional[str], + dump_scale_factors: typing.Optional[bool], + genome: typing.Optional[str], + bowtie2: typing.Optional[str], + gtf: typing.Optional[str], + gene_bed: typing.Optional[str], + blacklist: typing.Optional[str], + spikein_bowtie2: typing.Optional[str], + spikein_fasta: typing.Optional[str], + fasta: typing.Optional[LatchFile], + only_input: typing.Optional[bool], + only_genome: typing.Optional[bool], + only_preqc: typing.Optional[bool], + only_alignment: typing.Optional[bool], + only_filtering: typing.Optional[bool], + only_peak_calling: typing.Optional[bool], + skip_fastqc: typing.Optional[bool], + skip_trimming: typing.Optional[bool], + skip_removeduplicates: typing.Optional[bool], + skip_reporting: typing.Optional[bool], + skip_preseq: typing.Optional[bool], + skip_igv: typing.Optional[bool], + skip_dt_qc: typing.Optional[bool], + skip_heatmaps: typing.Optional[bool], + skip_peak_qc: typing.Optional[bool], + skip_multiqc: typing.Optional[bool], + remove_mitochondrial_reads: typing.Optional[bool], + mito_name: typing.Optional[str], + dedup_target_reads: typing.Optional[bool], + remove_linear_duplicates: typing.Optional[bool], + macs2_pvalue: typing.Optional[float], + singularity_pull_docker_container: typing.Optional[bool], + multiqc_methods_description: typing.Optional[str], + spikein_genome: typing.Optional[str], + clip_r1: typing.Optional[int], + clip_r2: typing.Optional[int], + three_prime_clip_r1: typing.Optional[int], + three_prime_clip_r2: typing.Optional[int], + trim_nextseq: typing.Optional[int], + minimum_alignment_q_score: typing.Optional[int], + end_to_end: typing.Optional[bool], + normalisation_mode: typing.Optional[str], + normalisation_binsize: typing.Optional[int], + peakcaller: typing.Optional[str], + use_control: typing.Optional[bool], + extend_fragments: typing.Optional[bool], + igg_scale_factor: typing.Optional[float], + seacr_peak_threshold: typing.Optional[float], + seacr_norm: typing.Optional[str], + seacr_stringent: typing.Optional[str], + macs2_qvalue: typing.Optional[float], + macs_gsize: typing.Optional[float], + macs2_narrow_peak: typing.Optional[bool], + macs2_broad_cutoff: typing.Optional[float], + consensus_peak_mode: typing.Optional[str], + replicate_threshold: typing.Optional[float], + igv_show_gene_names: typing.Optional[bool], + dt_qc_bam_binsize: typing.Optional[int], + dt_qc_corr_method: typing.Optional[str], + dt_heatmap_gene_beforelen: typing.Optional[int], + dt_heatmap_gene_bodylen: typing.Optional[int], + dt_heatmap_gene_afterlen: typing.Optional[int], + dt_heatmap_peak_beforelen: typing.Optional[int], + dt_heatmap_peak_afterlen: typing.Optional[int], + dt_calc_all_matrix: typing.Optional[bool], + min_frip_overlap: typing.Optional[float], + min_peak_overlap: typing.Optional[float], + igv_sort_by_groups: typing.Optional[bool], +) -> None: try: shared_dir = Path("/nf-workdir") - - ignore_list = [ "latch", ".latch", @@ -88,87 +158,87 @@ def nextflow_runtime(pvc_name: str, input: LatchFile, outdir: typing_extensions. "docker", "-c", "latch.config", - *get_flag('input', input), - *get_flag('outdir', outdir), - *get_flag('multiqc_title', multiqc_title), - *get_flag('save_reference', save_reference), - *get_flag('save_merged_fastq', save_merged_fastq), - *get_flag('save_trimmed', save_trimmed), - *get_flag('save_spikein_aligned', save_spikein_aligned), - *get_flag('save_unaligned', save_unaligned), - *get_flag('save_align_intermed', save_align_intermed), - *get_flag('email', email), - *get_flag('dump_scale_factors', dump_scale_factors), - *get_flag('genome', genome), - *get_flag('bowtie2', bowtie2), - *get_flag('gtf', gtf), - *get_flag('gene_bed', gene_bed), - *get_flag('blacklist', blacklist), - *get_flag('spikein_genome', spikein_genome), - *get_flag('spikein_bowtie2', spikein_bowtie2), - *get_flag('spikein_fasta', spikein_fasta), - *get_flag('fasta', fasta), - *get_flag('only_input', only_input), - *get_flag('only_genome', only_genome), - *get_flag('only_preqc', only_preqc), - *get_flag('only_alignment', only_alignment), - *get_flag('only_filtering', only_filtering), - *get_flag('only_peak_calling', only_peak_calling), - *get_flag('skip_fastqc', skip_fastqc), - *get_flag('skip_trimming', skip_trimming), - *get_flag('skip_removeduplicates', skip_removeduplicates), - *get_flag('skip_reporting', skip_reporting), - *get_flag('skip_preseq', skip_preseq), - *get_flag('skip_igv', skip_igv), - *get_flag('skip_dt_qc', skip_dt_qc), - *get_flag('skip_heatmaps', skip_heatmaps), - *get_flag('skip_peak_qc', skip_peak_qc), - *get_flag('skip_multiqc', skip_multiqc), - *get_flag('clip_r1', clip_r1), - *get_flag('clip_r2', clip_r2), - *get_flag('three_prime_clip_r1', three_prime_clip_r1), - *get_flag('three_prime_clip_r2', three_prime_clip_r2), - *get_flag('trim_nextseq', trim_nextseq), - *get_flag('minimum_alignment_q_score', minimum_alignment_q_score), - *get_flag('remove_mitochondrial_reads', remove_mitochondrial_reads), - *get_flag('mito_name', mito_name), - *get_flag('dedup_target_reads', dedup_target_reads), - *get_flag('remove_linear_duplicates', remove_linear_duplicates), - *get_flag('end_to_end', end_to_end), - *get_flag('normalisation_mode', normalisation_mode), - *get_flag('normalisation_binsize', normalisation_binsize), - *get_flag('peakcaller', peakcaller), - *get_flag('use_control', use_control), - *get_flag('extend_fragments', extend_fragments), - *get_flag('igg_scale_factor', igg_scale_factor), - *get_flag('seacr_peak_threshold', seacr_peak_threshold), - *get_flag('seacr_norm', seacr_norm), - *get_flag('seacr_stringent', seacr_stringent), - *get_flag('macs2_qvalue', macs2_qvalue), - *get_flag('macs2_pvalue', macs2_pvalue), - *get_flag('macs_gsize', macs_gsize), - *get_flag('macs2_narrow_peak', macs2_narrow_peak), - *get_flag('macs2_broad_cutoff', macs2_broad_cutoff), - *get_flag('consensus_peak_mode', consensus_peak_mode), - *get_flag('replicate_threshold', replicate_threshold), - *get_flag('igv_show_gene_names', igv_show_gene_names), - *get_flag('dt_qc_bam_binsize', dt_qc_bam_binsize), - *get_flag('dt_qc_corr_method', dt_qc_corr_method), - *get_flag('dt_heatmap_gene_beforelen', dt_heatmap_gene_beforelen), - *get_flag('dt_heatmap_gene_bodylen', dt_heatmap_gene_bodylen), - *get_flag('dt_heatmap_gene_afterlen', dt_heatmap_gene_afterlen), - *get_flag('dt_heatmap_peak_beforelen', dt_heatmap_peak_beforelen), - *get_flag('dt_heatmap_peak_afterlen', dt_heatmap_peak_afterlen), - *get_flag('dt_calc_all_matrix', dt_calc_all_matrix), - *get_flag('min_frip_overlap', min_frip_overlap), - *get_flag('min_peak_overlap', min_peak_overlap), - *get_flag('igv_sort_by_groups', igv_sort_by_groups), - *get_flag('singularity_pull_docker_container', singularity_pull_docker_container), - *get_flag('multiqc_methods_description', multiqc_methods_description) + *get_flag("input", input), + *get_flag("outdir", outdir), + *get_flag("multiqc_title", multiqc_title), + *get_flag("save_reference", save_reference), + *get_flag("save_merged_fastq", save_merged_fastq), + *get_flag("save_trimmed", save_trimmed), + *get_flag("save_spikein_aligned", save_spikein_aligned), + *get_flag("save_unaligned", save_unaligned), + *get_flag("save_align_intermed", save_align_intermed), + *get_flag("email", email), + *get_flag("dump_scale_factors", dump_scale_factors), + *get_flag("genome", genome), + *get_flag("bowtie2", bowtie2), + *get_flag("gtf", gtf), + *get_flag("gene_bed", gene_bed), + *get_flag("blacklist", blacklist), + *get_flag("spikein_genome", spikein_genome), + *get_flag("spikein_bowtie2", spikein_bowtie2), + *get_flag("spikein_fasta", spikein_fasta), + *get_flag("fasta", fasta), + *get_flag("only_input", only_input), + *get_flag("only_genome", only_genome), + *get_flag("only_preqc", only_preqc), + *get_flag("only_alignment", only_alignment), + *get_flag("only_filtering", only_filtering), + *get_flag("only_peak_calling", only_peak_calling), + *get_flag("skip_fastqc", skip_fastqc), + *get_flag("skip_trimming", skip_trimming), + *get_flag("skip_removeduplicates", skip_removeduplicates), + *get_flag("skip_reporting", skip_reporting), + *get_flag("skip_preseq", skip_preseq), + *get_flag("skip_igv", skip_igv), + *get_flag("skip_dt_qc", skip_dt_qc), + *get_flag("skip_heatmaps", skip_heatmaps), + *get_flag("skip_peak_qc", skip_peak_qc), + *get_flag("skip_multiqc", skip_multiqc), + *get_flag("clip_r1", clip_r1), + *get_flag("clip_r2", clip_r2), + *get_flag("three_prime_clip_r1", three_prime_clip_r1), + *get_flag("three_prime_clip_r2", three_prime_clip_r2), + *get_flag("trim_nextseq", trim_nextseq), + *get_flag("minimum_alignment_q_score", minimum_alignment_q_score), + *get_flag("remove_mitochondrial_reads", remove_mitochondrial_reads), + *get_flag("mito_name", mito_name), + *get_flag("dedup_target_reads", dedup_target_reads), + *get_flag("remove_linear_duplicates", remove_linear_duplicates), + *get_flag("end_to_end", end_to_end), + *get_flag("normalisation_mode", normalisation_mode), + *get_flag("normalisation_binsize", normalisation_binsize), + *get_flag("peakcaller", peakcaller), + *get_flag("use_control", use_control), + *get_flag("extend_fragments", extend_fragments), + *get_flag("igg_scale_factor", igg_scale_factor), + *get_flag("seacr_peak_threshold", seacr_peak_threshold), + *get_flag("seacr_norm", seacr_norm), + *get_flag("seacr_stringent", seacr_stringent), + *get_flag("macs2_qvalue", macs2_qvalue), + *get_flag("macs2_pvalue", macs2_pvalue), + *get_flag("macs_gsize", macs_gsize), + *get_flag("macs2_narrow_peak", macs2_narrow_peak), + *get_flag("macs2_broad_cutoff", macs2_broad_cutoff), + *get_flag("consensus_peak_mode", consensus_peak_mode), + *get_flag("replicate_threshold", replicate_threshold), + *get_flag("igv_show_gene_names", igv_show_gene_names), + *get_flag("dt_qc_bam_binsize", dt_qc_bam_binsize), + *get_flag("dt_qc_corr_method", dt_qc_corr_method), + *get_flag("dt_heatmap_gene_beforelen", dt_heatmap_gene_beforelen), + *get_flag("dt_heatmap_gene_bodylen", dt_heatmap_gene_bodylen), + *get_flag("dt_heatmap_gene_afterlen", dt_heatmap_gene_afterlen), + *get_flag("dt_heatmap_peak_beforelen", dt_heatmap_peak_beforelen), + *get_flag("dt_heatmap_peak_afterlen", dt_heatmap_peak_afterlen), + *get_flag("dt_calc_all_matrix", dt_calc_all_matrix), + *get_flag("min_frip_overlap", min_frip_overlap), + *get_flag("min_peak_overlap", min_peak_overlap), + *get_flag("igv_sort_by_groups", igv_sort_by_groups), + *get_flag("singularity_pull_docker_container", singularity_pull_docker_container), + *get_flag("multiqc_methods_description", multiqc_methods_description), ] print("Launching Nextflow Runtime") - print(' '.join(cmd)) + print(" ".join(cmd)) print(flush=True) env = { @@ -198,9 +268,86 @@ def nextflow_runtime(pvc_name: str, input: LatchFile, outdir: typing_extensions. remote.upload_from(nextflow_log) - @workflow(metadata._nextflow_metadata) -def nf_nf_core_cutandrun(input: LatchFile, outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], multiqc_title: typing.Optional[str], save_reference: typing.Optional[bool], save_merged_fastq: typing.Optional[bool], save_trimmed: typing.Optional[bool], save_spikein_aligned: typing.Optional[bool], save_unaligned: typing.Optional[bool], save_align_intermed: typing.Optional[bool], email: typing.Optional[str], dump_scale_factors: typing.Optional[bool], genome: typing.Optional[str], bowtie2: typing.Optional[str], gtf: typing.Optional[str], gene_bed: typing.Optional[str], blacklist: typing.Optional[str], spikein_bowtie2: typing.Optional[str], spikein_fasta: typing.Optional[str], fasta: typing.Optional[LatchFile], only_input: typing.Optional[bool], only_genome: typing.Optional[bool], only_preqc: typing.Optional[bool], only_alignment: typing.Optional[bool], only_filtering: typing.Optional[bool], only_peak_calling: typing.Optional[bool], skip_fastqc: typing.Optional[bool], skip_trimming: typing.Optional[bool], skip_removeduplicates: typing.Optional[bool], skip_reporting: typing.Optional[bool], skip_preseq: typing.Optional[bool], skip_igv: typing.Optional[bool], skip_dt_qc: typing.Optional[bool], skip_heatmaps: typing.Optional[bool], skip_peak_qc: typing.Optional[bool], skip_multiqc: typing.Optional[bool], remove_mitochondrial_reads: typing.Optional[bool], mito_name: typing.Optional[str], dedup_target_reads: typing.Optional[bool], remove_linear_duplicates: typing.Optional[bool], macs2_pvalue: typing.Optional[float], singularity_pull_docker_container: typing.Optional[bool], multiqc_methods_description: typing.Optional[str], spikein_genome: typing.Optional[str] = 'K12-MG1655', clip_r1: typing.Optional[int] = 0, clip_r2: typing.Optional[int] = 0, three_prime_clip_r1: typing.Optional[int] = 0, three_prime_clip_r2: typing.Optional[int] = 0, trim_nextseq: typing.Optional[int] = 0, minimum_alignment_q_score: typing.Optional[int] = 20, end_to_end: typing.Optional[bool] = True, normalisation_mode: typing.Optional[str] = 'Spikein', normalisation_binsize: typing.Optional[int] = 50, peakcaller: typing.Optional[str] = 'seacr', use_control: typing.Optional[bool] = True, extend_fragments: typing.Optional[bool] = True, igg_scale_factor: typing.Optional[float] = 0.5, seacr_peak_threshold: typing.Optional[float] = 0.05, seacr_norm: typing.Optional[str] = 'non', seacr_stringent: typing.Optional[str] = 'stringent', macs2_qvalue: typing.Optional[float] = 0.01, macs_gsize: typing.Optional[float] = 2700000000, macs2_narrow_peak: typing.Optional[bool] = True, macs2_broad_cutoff: typing.Optional[float] = 0.1, consensus_peak_mode: typing.Optional[str] = 'group', replicate_threshold: typing.Optional[float] = 1, igv_show_gene_names: typing.Optional[bool] = True, dt_qc_bam_binsize: typing.Optional[int] = 500, dt_qc_corr_method: typing.Optional[str] = 'pearson', dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, dt_calc_all_matrix: typing.Optional[bool] = True, min_frip_overlap: typing.Optional[float] = 0.2, min_peak_overlap: typing.Optional[float] = 0.2, igv_sort_by_groups: typing.Optional[bool] = True) -> None: +def nf_nf_core_cutandrun( + input: LatchFile, + outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], + multiqc_title: typing.Optional[str], + save_reference: typing.Optional[bool], + save_merged_fastq: typing.Optional[bool], + save_trimmed: typing.Optional[bool], + save_spikein_aligned: typing.Optional[bool], + save_unaligned: typing.Optional[bool], + save_align_intermed: typing.Optional[bool], + email: typing.Optional[str], + dump_scale_factors: typing.Optional[bool], + genome: typing.Optional[str], + bowtie2: typing.Optional[str], + gtf: typing.Optional[str], + gene_bed: typing.Optional[str], + blacklist: typing.Optional[str], + spikein_bowtie2: typing.Optional[str], + spikein_fasta: typing.Optional[str], + fasta: typing.Optional[LatchFile], + only_input: typing.Optional[bool], + only_genome: typing.Optional[bool], + only_preqc: typing.Optional[bool], + only_alignment: typing.Optional[bool], + only_filtering: typing.Optional[bool], + only_peak_calling: typing.Optional[bool], + skip_fastqc: typing.Optional[bool], + skip_trimming: typing.Optional[bool], + skip_removeduplicates: typing.Optional[bool], + skip_reporting: typing.Optional[bool], + skip_preseq: typing.Optional[bool], + skip_igv: typing.Optional[bool], + skip_dt_qc: typing.Optional[bool], + skip_heatmaps: typing.Optional[bool], + skip_peak_qc: typing.Optional[bool], + skip_multiqc: typing.Optional[bool], + remove_mitochondrial_reads: typing.Optional[bool], + mito_name: typing.Optional[str], + dedup_target_reads: typing.Optional[bool], + remove_linear_duplicates: typing.Optional[bool], + macs2_pvalue: typing.Optional[float], + singularity_pull_docker_container: typing.Optional[bool], + multiqc_methods_description: typing.Optional[str], + spikein_genome: typing.Optional[str] = "K12-MG1655", + clip_r1: typing.Optional[int] = 0, + clip_r2: typing.Optional[int] = 0, + three_prime_clip_r1: typing.Optional[int] = 0, + three_prime_clip_r2: typing.Optional[int] = 0, + trim_nextseq: typing.Optional[int] = 0, + minimum_alignment_q_score: typing.Optional[int] = 20, + end_to_end: typing.Optional[bool] = True, + normalisation_mode: typing.Optional[str] = "Spikein", + normalisation_binsize: typing.Optional[int] = 50, + peakcaller: typing.Optional[str] = "seacr", + use_control: typing.Optional[bool] = True, + extend_fragments: typing.Optional[bool] = True, + igg_scale_factor: typing.Optional[float] = 0.5, + seacr_peak_threshold: typing.Optional[float] = 0.05, + seacr_norm: typing.Optional[str] = "non", + seacr_stringent: typing.Optional[str] = "stringent", + macs2_qvalue: typing.Optional[float] = 0.01, + macs_gsize: typing.Optional[float] = 2700000000.0, + macs2_narrow_peak: typing.Optional[bool] = True, + macs2_broad_cutoff: typing.Optional[float] = 0.1, + consensus_peak_mode: typing.Optional[str] = "group", + replicate_threshold: typing.Optional[float] = 1.0, + igv_show_gene_names: typing.Optional[bool] = True, + dt_qc_bam_binsize: typing.Optional[int] = 500, + dt_qc_corr_method: typing.Optional[str] = "pearson", + dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, + dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, + dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, + dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, + dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, + dt_calc_all_matrix: typing.Optional[bool] = True, + min_frip_overlap: typing.Optional[float] = 0.2, + min_peak_overlap: typing.Optional[float] = 0.2, + igv_sort_by_groups: typing.Optional[bool] = True, +) -> None: """ nf-core/cutandrun @@ -208,5 +355,83 @@ def nf_nf_core_cutandrun(input: LatchFile, outdir: typing_extensions.Annotated[L """ pvc_name: str = initialize() - nextflow_runtime(pvc_name=pvc_name, input=input, outdir=outdir, multiqc_title=multiqc_title, save_reference=save_reference, save_merged_fastq=save_merged_fastq, save_trimmed=save_trimmed, save_spikein_aligned=save_spikein_aligned, save_unaligned=save_unaligned, save_align_intermed=save_align_intermed, email=email, dump_scale_factors=dump_scale_factors, genome=genome, bowtie2=bowtie2, gtf=gtf, gene_bed=gene_bed, blacklist=blacklist, spikein_genome=spikein_genome, spikein_bowtie2=spikein_bowtie2, spikein_fasta=spikein_fasta, fasta=fasta, only_input=only_input, only_genome=only_genome, only_preqc=only_preqc, only_alignment=only_alignment, only_filtering=only_filtering, only_peak_calling=only_peak_calling, skip_fastqc=skip_fastqc, skip_trimming=skip_trimming, skip_removeduplicates=skip_removeduplicates, skip_reporting=skip_reporting, skip_preseq=skip_preseq, skip_igv=skip_igv, skip_dt_qc=skip_dt_qc, skip_heatmaps=skip_heatmaps, skip_peak_qc=skip_peak_qc, skip_multiqc=skip_multiqc, clip_r1=clip_r1, clip_r2=clip_r2, three_prime_clip_r1=three_prime_clip_r1, three_prime_clip_r2=three_prime_clip_r2, trim_nextseq=trim_nextseq, minimum_alignment_q_score=minimum_alignment_q_score, remove_mitochondrial_reads=remove_mitochondrial_reads, mito_name=mito_name, dedup_target_reads=dedup_target_reads, remove_linear_duplicates=remove_linear_duplicates, end_to_end=end_to_end, normalisation_mode=normalisation_mode, normalisation_binsize=normalisation_binsize, peakcaller=peakcaller, use_control=use_control, extend_fragments=extend_fragments, igg_scale_factor=igg_scale_factor, seacr_peak_threshold=seacr_peak_threshold, seacr_norm=seacr_norm, seacr_stringent=seacr_stringent, macs2_qvalue=macs2_qvalue, macs2_pvalue=macs2_pvalue, macs_gsize=macs_gsize, macs2_narrow_peak=macs2_narrow_peak, macs2_broad_cutoff=macs2_broad_cutoff, consensus_peak_mode=consensus_peak_mode, replicate_threshold=replicate_threshold, igv_show_gene_names=igv_show_gene_names, dt_qc_bam_binsize=dt_qc_bam_binsize, dt_qc_corr_method=dt_qc_corr_method, dt_heatmap_gene_beforelen=dt_heatmap_gene_beforelen, dt_heatmap_gene_bodylen=dt_heatmap_gene_bodylen, dt_heatmap_gene_afterlen=dt_heatmap_gene_afterlen, dt_heatmap_peak_beforelen=dt_heatmap_peak_beforelen, dt_heatmap_peak_afterlen=dt_heatmap_peak_afterlen, dt_calc_all_matrix=dt_calc_all_matrix, min_frip_overlap=min_frip_overlap, min_peak_overlap=min_peak_overlap, igv_sort_by_groups=igv_sort_by_groups, singularity_pull_docker_container=singularity_pull_docker_container, multiqc_methods_description=multiqc_methods_description) - + nextflow_runtime( + pvc_name=pvc_name, + input=input, + outdir=outdir, + multiqc_title=multiqc_title, + save_reference=save_reference, + save_merged_fastq=save_merged_fastq, + save_trimmed=save_trimmed, + save_spikein_aligned=save_spikein_aligned, + save_unaligned=save_unaligned, + save_align_intermed=save_align_intermed, + email=email, + dump_scale_factors=dump_scale_factors, + genome=genome, + bowtie2=bowtie2, + gtf=gtf, + gene_bed=gene_bed, + blacklist=blacklist, + spikein_genome=spikein_genome, + spikein_bowtie2=spikein_bowtie2, + spikein_fasta=spikein_fasta, + fasta=fasta, + only_input=only_input, + only_genome=only_genome, + only_preqc=only_preqc, + only_alignment=only_alignment, + only_filtering=only_filtering, + only_peak_calling=only_peak_calling, + skip_fastqc=skip_fastqc, + skip_trimming=skip_trimming, + skip_removeduplicates=skip_removeduplicates, + skip_reporting=skip_reporting, + skip_preseq=skip_preseq, + skip_igv=skip_igv, + skip_dt_qc=skip_dt_qc, + skip_heatmaps=skip_heatmaps, + skip_peak_qc=skip_peak_qc, + skip_multiqc=skip_multiqc, + clip_r1=clip_r1, + clip_r2=clip_r2, + three_prime_clip_r1=three_prime_clip_r1, + three_prime_clip_r2=three_prime_clip_r2, + trim_nextseq=trim_nextseq, + minimum_alignment_q_score=minimum_alignment_q_score, + remove_mitochondrial_reads=remove_mitochondrial_reads, + mito_name=mito_name, + dedup_target_reads=dedup_target_reads, + remove_linear_duplicates=remove_linear_duplicates, + end_to_end=end_to_end, + normalisation_mode=normalisation_mode, + normalisation_binsize=normalisation_binsize, + peakcaller=peakcaller, + use_control=use_control, + extend_fragments=extend_fragments, + igg_scale_factor=igg_scale_factor, + seacr_peak_threshold=seacr_peak_threshold, + seacr_norm=seacr_norm, + seacr_stringent=seacr_stringent, + macs2_qvalue=macs2_qvalue, + macs2_pvalue=macs2_pvalue, + macs_gsize=macs_gsize, + macs2_narrow_peak=macs2_narrow_peak, + macs2_broad_cutoff=macs2_broad_cutoff, + consensus_peak_mode=consensus_peak_mode, + replicate_threshold=replicate_threshold, + igv_show_gene_names=igv_show_gene_names, + dt_qc_bam_binsize=dt_qc_bam_binsize, + dt_qc_corr_method=dt_qc_corr_method, + dt_heatmap_gene_beforelen=dt_heatmap_gene_beforelen, + dt_heatmap_gene_bodylen=dt_heatmap_gene_bodylen, + dt_heatmap_gene_afterlen=dt_heatmap_gene_afterlen, + dt_heatmap_peak_beforelen=dt_heatmap_peak_beforelen, + dt_heatmap_peak_afterlen=dt_heatmap_peak_afterlen, + dt_calc_all_matrix=dt_calc_all_matrix, + min_frip_overlap=min_frip_overlap, + min_peak_overlap=min_peak_overlap, + igv_sort_by_groups=igv_sort_by_groups, + singularity_pull_docker_container=singularity_pull_docker_container, + multiqc_methods_description=multiqc_methods_description, + ) From 139cd4837a4befcea0e1c1e54eb3650bda351386 Mon Sep 17 00:00:00 2001 From: Harihara Date: Sat, 14 Sep 2024 10:04:53 -0700 Subject: [PATCH 3/5] Fixed flows, launch plans --- .../docker-build-logs.txt | 109 ---- .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 37 +- .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ .../docker-build-logs.txt | 84 +++ Dockerfile => .latch/Dockerfile | 11 +- .latch/bin/nextflow | 458 --------------- .latch/config | 2 +- .latch/nextflow_version | 1 - .latch/workflow_name | 2 +- latch_metadata/__init__.py | 164 +++++- latch_metadata/parameters.py | 544 ++++++++++------- wf/__init__.py | 254 ++++++++ wf/entrypoint.py | 555 ++++++++---------- 44 files changed, 3753 insertions(+), 1156 deletions(-) delete mode 100644 .latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-04e31f/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-0bd102/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-14b20d/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-15bc6c/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-1bcd35/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-28232c/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-2d9182/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-351d10/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-4c5052/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-56df58/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-57ae63/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-71d084/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-763f25/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-865dcb/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9567ca/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-96dd33/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-987a0c/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9e0fe1/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ad261b/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b518a6/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b7129c/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-bf27db/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-c42ac6/docker-build-logs.txt rename .latch/.logs/{35597_nf_nf_core_cutandrun:0.1.0-88a084 => 35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-cba6dd}/docker-build-logs.txt (67%) create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d14dd7/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-db89d4/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ddb034/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e3cb14/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e9d826/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ebd0e4/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ecbaf0/docker-build-logs.txt create mode 100644 .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-f07047/docker-build-logs.txt create mode 100644 .latch/.logs/35929_nf_nf_core_cutandrun:0.1.0-96a42f-wip-da774b/docker-build-logs.txt create mode 100644 .latch/.logs/36340_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d6e2a4/docker-build-logs.txt rename Dockerfile => .latch/Dockerfile (65%) delete mode 100755 .latch/bin/nextflow delete mode 100644 .latch/nextflow_version create mode 100644 wf/__init__.py diff --git a/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt b/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt deleted file mode 100644 index c5ff52dc..00000000 --- a/.latch/.logs/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250/docker-build-logs.txt +++ /dev/null @@ -1,109 +0,0 @@ -Step 1/15 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main - - - ---> f23c37efcd53 - -Step 2/15 : workdir /tmp/docker-build/work/ - - - ---> Using cache - - ---> dd16c2d4f334 - -Step 3/15 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] - - - ---> Using cache - - ---> aaada0c60239 - -Step 4/15 : env TZ='Etc/UTC' - - - ---> Using cache - - ---> d6869bc03deb - -Step 5/15 : env LANG='en_US.UTF-8' - - - ---> Using cache - - ---> ae2f7ca65880 - -Step 6/15 : arg DEBIAN_FRONTEND=noninteractive - - - ---> Using cache - - ---> f029b3f36a86 - -Step 7/15 : run pip install latch==2.46.6 - - - ---> Using cache - - ---> be1350f11537 - -Step 8/15 : run mkdir /opt/latch - - - ---> Using cache - - ---> e8dd4419614b - -Step 9/15 : run apt-get update && apt-get install -y default-jre-headless - - - ---> Using cache - - ---> 447cbd90cbc9 - -Step 10/15 : copy . /root/ - - - ---> a9468c7b7bff - -Step 11/15 : run ln -s /root/.latch/bin/nextflow /root/nextflow - - - ---> Running in d056c3d7bd82 - -ln -s /root/.latch/bin/nextflow /root/nextflow - - ---> 8bb36367c2d8 - -Step 12/15 : run ln -s /root/.latch/.nextflow /root/.nextflow - - - ---> Running in 1abcae1835aa - -ln -s /root/.latch/.nextflow /root/.nextflow - - ---> 50622da6d862 - -Step 13/15 : arg tag - - - ---> Running in ebadde0805f8 - - ---> c030684dc96f - -Step 14/15 : env FLYTE_INTERNAL_IMAGE $tag - - - ---> Running in 22e9ae745493 - - ---> ce25eee14447 - -Step 15/15 : workdir /root - - - ---> Running in eec1c7ae0d2d - - ---> 2f082b780475 - -Successfully built 2f082b780475 - -Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35597_wf_entrypoint_nf_nf_core_cutandrun:0.1.0-dc9250 - diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-04e31f/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-04e31f/docker-build-logs.txt new file mode 100644 index 00000000..0e3b9be1 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-04e31f/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 3130bdb47e3b + +Step 10/12 : arg tag + + + ---> Running in 1a3875c24100 + + ---> 81ecc678fa8c + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in a1e153766787 + + ---> 1db50306d0e6 + +Step 12/12 : workdir /root + + + ---> Running in b5b897a3da23 + + ---> 230f37d31b1c + +Successfully built 230f37d31b1c + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-04e31f + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-0bd102/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-0bd102/docker-build-logs.txt new file mode 100644 index 00000000..7677a393 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-0bd102/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> dad6ab07ea8c + +Step 10/12 : arg tag + + + ---> Running in 7d31976c569c + + ---> b1c55a307188 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 5fa81e1e8c1f + + ---> d39c058d6be3 + +Step 12/12 : workdir /root + + + ---> Running in 04fa31636ed1 + + ---> d2fd0276f2d2 + +Successfully built d2fd0276f2d2 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-0bd102 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-14b20d/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-14b20d/docker-build-logs.txt new file mode 100644 index 00000000..d95a8806 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-14b20d/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 2b71d40a148e + +Step 10/12 : arg tag + + + ---> Running in b16d629ad398 + + ---> ea84f03714a6 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 29ebfa3ab767 + + ---> 29264892e6b9 + +Step 12/12 : workdir /root + + + ---> Running in 90cfd5010b9d + + ---> 3a019d9bf388 + +Successfully built 3a019d9bf388 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-14b20d + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-15bc6c/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-15bc6c/docker-build-logs.txt new file mode 100644 index 00000000..b670ad79 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-15bc6c/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 9e03c04bae9d + +Step 10/12 : arg tag + + + ---> Running in cdd6af3b7e8e + + ---> 6d7e0cacd835 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in f1a23b89a2a7 + + ---> 3fce6f74d6eb + +Step 12/12 : workdir /root + + + ---> Running in 712abffd8a6e + + ---> 6003ee35c04d + +Successfully built 6003ee35c04d + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-15bc6c + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-1bcd35/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-1bcd35/docker-build-logs.txt new file mode 100644 index 00000000..1f1a2f53 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-1bcd35/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 0cf27bb31ca3 + +Step 10/12 : arg tag + + + ---> Running in e0dc8b7faa3c + + ---> 0da3c7991f84 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 3ea16f201802 + + ---> 04f5e600c34a + +Step 12/12 : workdir /root + + + ---> Running in e8fc1b0af7ee + + ---> 69c8d27bff4c + +Successfully built 69c8d27bff4c + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-1bcd35 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-28232c/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-28232c/docker-build-logs.txt new file mode 100644 index 00000000..503dc02d --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-28232c/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> df6c61763974 + +Step 10/12 : arg tag + + + ---> Running in 1655d764151f + + ---> d5ac8ad639d8 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 6e67844d526c + + ---> 4321d2715fe1 + +Step 12/12 : workdir /root + + + ---> Running in 9fc7673c082e + + ---> 376c3da17a89 + +Successfully built 376c3da17a89 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-28232c + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-2d9182/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-2d9182/docker-build-logs.txt new file mode 100644 index 00000000..3ca8bd1a --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-2d9182/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 86a3e7c47da1 + +Step 10/12 : arg tag + + + ---> Running in ae1ba8af9dc1 + + ---> ad7fc24753e2 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 9aa1737e3715 + + ---> 6a435cb9668d + +Step 12/12 : workdir /root + + + ---> Running in aeec25ed1a37 + + ---> d1781653c28c + +Successfully built d1781653c28c + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-2d9182 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-351d10/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-351d10/docker-build-logs.txt new file mode 100644 index 00000000..7b3ca337 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-351d10/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 9e902eae1ea3 + +Step 10/12 : arg tag + + + ---> Running in 9d14e5bfa7e3 + + ---> a9603d3d47d5 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 59d4e38914de + + ---> 85ac530a6088 + +Step 12/12 : workdir /root + + + ---> Running in 6329d55db208 + + ---> 0625de06e6b6 + +Successfully built 0625de06e6b6 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-351d10 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-4c5052/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-4c5052/docker-build-logs.txt new file mode 100644 index 00000000..9daa074b --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-4c5052/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> d56a4a17371a + +Step 10/12 : arg tag + + + ---> Running in 59b51360e9d2 + + ---> 40baa7e11f61 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in ccefa83d7447 + + ---> 406577834233 + +Step 12/12 : workdir /root + + + ---> Running in 0d74d18aa13c + + ---> 5135d419a599 + +Successfully built 5135d419a599 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-4c5052 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-56df58/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-56df58/docker-build-logs.txt new file mode 100644 index 00000000..48961901 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-56df58/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> fd5ff9c9acca + +Step 10/12 : arg tag + + + ---> Running in f3e620f8173b + + ---> 0bf3e95c8497 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 7c27dc3d39d0 + + ---> a8fafea98738 + +Step 12/12 : workdir /root + + + ---> Running in da1ce8022b3f + + ---> 20c8e1b175ba + +Successfully built 20c8e1b175ba + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-56df58 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-57ae63/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-57ae63/docker-build-logs.txt new file mode 100644 index 00000000..5eaeb541 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-57ae63/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 6498c04066c2 + +Step 10/12 : arg tag + + + ---> Running in 85506ce9ccb0 + + ---> f30f694fad09 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 77f9c7f10e3b + + ---> 82ad3122a40a + +Step 12/12 : workdir /root + + + ---> Running in da469e851fae + + ---> 059f9d5e2015 + +Successfully built 059f9d5e2015 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-57ae63 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-71d084/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-71d084/docker-build-logs.txt new file mode 100644 index 00000000..0c105787 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-71d084/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> ab50ac925e84 + +Step 10/12 : arg tag + + + ---> Running in 08c9a5181214 + + ---> a9ce6e06c1b2 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in f4f18366d429 + + ---> c86d559b66ae + +Step 12/12 : workdir /root + + + ---> Running in c74ea9314522 + + ---> aecc8ad07262 + +Successfully built aecc8ad07262 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-71d084 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-763f25/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-763f25/docker-build-logs.txt new file mode 100644 index 00000000..253473e6 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-763f25/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 948f188bc47e + +Step 10/12 : arg tag + + + ---> Running in 0d435207e1e4 + + ---> cf42fa807751 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in dd3d64721195 + + ---> e4d30c90f749 + +Step 12/12 : workdir /root + + + ---> Running in 44fff340fede + + ---> 0750b4e836ee + +Successfully built 0750b4e836ee + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-763f25 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-865dcb/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-865dcb/docker-build-logs.txt new file mode 100644 index 00000000..409966ae --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-865dcb/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v1.1.7 + + + ---> 884f8e5666b5 + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 6b89f6228cb4 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> 080efe1dfb73 + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> e7bc60ad5baf + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> e133f72562f7 + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 85d9823692d7 + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> 502514529836 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 4fdeaac6d35e + +Step 9/12 : copy . /root/ + + + ---> 7fb99a22e6d8 + +Step 10/12 : arg tag + + + ---> Running in b0733796081d + + ---> 0d641350b361 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 61875c7c145d + + ---> ebbfcb4bb9f9 + +Step 12/12 : workdir /root + + + ---> Running in c7f8d81359f8 + + ---> 7c557785afa8 + +Successfully built 7c557785afa8 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-865dcb + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9567ca/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9567ca/docker-build-logs.txt new file mode 100644 index 00000000..86cf366c --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9567ca/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 83c2c1234d8f + +Step 10/12 : arg tag + + + ---> Running in ebd760eba94a + + ---> de28a96e6bfa + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 355dc2a6602f + + ---> 32824d2075be + +Step 12/12 : workdir /root + + + ---> Running in 3910a6e5640a + + ---> 234417e917aa + +Successfully built 234417e917aa + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9567ca + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-96dd33/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-96dd33/docker-build-logs.txt new file mode 100644 index 00000000..73a48bd5 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-96dd33/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> a54c7a5429ce + +Step 10/12 : arg tag + + + ---> Running in 1611dd9279c6 + + ---> 47275525bde6 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 6c382ae94c87 + + ---> b0392c15561b + +Step 12/12 : workdir /root + + + ---> Running in 30c93e051f0e + + ---> d4c27b80d980 + +Successfully built d4c27b80d980 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-96dd33 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-987a0c/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-987a0c/docker-build-logs.txt new file mode 100644 index 00000000..8457eb41 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-987a0c/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 80c0c7de0ca1 + +Step 10/12 : arg tag + + + ---> Running in d4a11d3c466c + + ---> 307c0bfb46f1 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in cdea3d937214 + + ---> c06512a702a1 + +Step 12/12 : workdir /root + + + ---> Running in 82d651df758b + + ---> e79a074ad077 + +Successfully built e79a074ad077 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-987a0c + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9e0fe1/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9e0fe1/docker-build-logs.txt new file mode 100644 index 00000000..1f500d2d --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9e0fe1/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> da49a31cd89a + +Step 10/12 : arg tag + + + ---> Running in abb202537cc8 + + ---> ea17cc31efec + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in e9fbc06d136d + + ---> 8c0281c60154 + +Step 12/12 : workdir /root + + + ---> Running in bab07333531b + + ---> 284ec5d8892f + +Successfully built 284ec5d8892f + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-9e0fe1 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ad261b/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ad261b/docker-build-logs.txt new file mode 100644 index 00000000..e1550004 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ad261b/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v1.1.7 + + + ---> 884f8e5666b5 + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 6b89f6228cb4 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> 080efe1dfb73 + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> e7bc60ad5baf + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> e133f72562f7 + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 85d9823692d7 + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> 502514529836 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 4fdeaac6d35e + +Step 9/12 : copy . /root/ + + + ---> 68e7c42af2d2 + +Step 10/12 : arg tag + + + ---> Running in 5de5a6584537 + + ---> 839ef0abb1ed + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in dc7144ebbcae + + ---> f98bfd798675 + +Step 12/12 : workdir /root + + + ---> Running in 880b42e59ec1 + + ---> 44927ce39de0 + +Successfully built 44927ce39de0 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ad261b + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b518a6/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b518a6/docker-build-logs.txt new file mode 100644 index 00000000..854aebe6 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b518a6/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> ed8ee87027ef + +Step 10/12 : arg tag + + + ---> Running in 40213fe84693 + + ---> 9148e50c4420 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in e7368bf4feda + + ---> 2edcdd39e86a + +Step 12/12 : workdir /root + + + ---> Running in 88d0bd161d4b + + ---> aee832deebf2 + +Successfully built aee832deebf2 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b518a6 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b7129c/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b7129c/docker-build-logs.txt new file mode 100644 index 00000000..cd306bc1 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b7129c/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 5202980fdfb1 + +Step 10/12 : arg tag + + + ---> Running in 1c5b68e98fbe + + ---> 844b80cb292d + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 0901e39d7535 + + ---> 5f8bcc67ec30 + +Step 12/12 : workdir /root + + + ---> Running in 72f7b313cdf3 + + ---> 95e59381e04f + +Successfully built 95e59381e04f + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-b7129c + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-bf27db/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-bf27db/docker-build-logs.txt new file mode 100644 index 00000000..ae48e390 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-bf27db/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> fb87264e5e17 + +Step 10/12 : arg tag + + + ---> Running in 2f8f5e5a25e6 + + ---> 6f0e0e7ac98c + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 3a3b32bd4042 + + ---> 71bd7b5475b0 + +Step 12/12 : workdir /root + + + ---> Running in 668b5d8572a1 + + ---> bf9ee4dbd608 + +Successfully built bf9ee4dbd608 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-bf27db + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-c42ac6/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-c42ac6/docker-build-logs.txt new file mode 100644 index 00000000..11f1a9ad --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-c42ac6/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 6b2a58419862 + +Step 10/12 : arg tag + + + ---> Running in f08de6a53165 + + ---> d9b4b6ed986b + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 7384af432323 + + ---> 6c221db07603 + +Step 12/12 : workdir /root + + + ---> Running in d41636d08961 + + ---> 65bf49e78c98 + +Successfully built 65bf49e78c98 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-c42ac6 + diff --git a/.latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-cba6dd/docker-build-logs.txt similarity index 67% rename from .latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt rename to .latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-cba6dd/docker-build-logs.txt index a1c2a33e..ed1de1be 100644 --- a/.latch/.logs/35597_nf_nf_core_cutandrun:0.1.0-88a084/docker-build-logs.txt +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-cba6dd/docker-build-logs.txt @@ -62,48 +62,19 @@ Step 9/15 : run apt-get update && apt-get install -y default-jre-headless Step 10/15 : copy . /root/ - ---> 3f3e4ac9e1cf + ---> 58b86d8aeb3f Step 11/15 : run ln -s /root/.latch/bin/nextflow /root/nextflow - ---> Running in c15dac84f040 + ---> Running in 8d5137feef53 ln -s /root/.latch/bin/nextflow /root/nextflow  - ---> c24b956d7b18 + ---> 13a37c04603e Step 12/15 : run ln -s /root/.latch/.nextflow /root/.nextflow - ---> Running in 72a50362b6e5 - -ln -s /root/.latch/.nextflow /root/.nextflow - - ---> b64ac576009f - -Step 13/15 : arg tag - - - ---> Running in 235d45699fc3 - - ---> c1de3bbb49b5 - -Step 14/15 : env FLYTE_INTERNAL_IMAGE $tag - - - ---> Running in 7662e8e183e9 - - ---> 11282b33ea19 - -Step 15/15 : workdir /root - - - ---> Running in b851b30539aa - - ---> a2635d95d5dd - -Successfully built a2635d95d5dd - -Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35597_nf_nf_core_cutandrun:0.1.0-88a084 + ---> Running in 4bae33b8f040 diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d14dd7/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d14dd7/docker-build-logs.txt new file mode 100644 index 00000000..5364ad69 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d14dd7/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v1.1.7 + + + ---> 884f8e5666b5 + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 6b89f6228cb4 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> 080efe1dfb73 + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> e7bc60ad5baf + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> e133f72562f7 + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 85d9823692d7 + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> 502514529836 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 4fdeaac6d35e + +Step 9/12 : copy . /root/ + + + ---> 25923086db67 + +Step 10/12 : arg tag + + + ---> Running in 0cdf7eb3a530 + + ---> 5fcdc30e5af1 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 84e637cb6492 + + ---> 19d31264f1c9 + +Step 12/12 : workdir /root + + + ---> Running in f2eb7b9dd166 + + ---> 5775a2145175 + +Successfully built 5775a2145175 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d14dd7 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-db89d4/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-db89d4/docker-build-logs.txt new file mode 100644 index 00000000..0c2fd0bf --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-db89d4/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> eb2c2fb37151 + +Step 10/12 : arg tag + + + ---> Running in 25bd95ce489d + + ---> f100e8ae53c5 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 9bc6163eed9c + + ---> 8fb41777054e + +Step 12/12 : workdir /root + + + ---> Running in 1d59e237aaea + + ---> c0f4d089b410 + +Successfully built c0f4d089b410 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-db89d4 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ddb034/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ddb034/docker-build-logs.txt new file mode 100644 index 00000000..57c6239c --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ddb034/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 23f11d8c7f4f + +Step 10/12 : arg tag + + + ---> Running in 41abaeb60b2f + + ---> 7e2cba0ae234 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in f4a0f5372c9a + + ---> 2fed004bf78f + +Step 12/12 : workdir /root + + + ---> Running in 70f2d67cfc80 + + ---> 972d51653358 + +Successfully built 972d51653358 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ddb034 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e3cb14/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e3cb14/docker-build-logs.txt new file mode 100644 index 00000000..d5435c2c --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e3cb14/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 1c732fdf2616 + +Step 10/12 : arg tag + + + ---> Running in 68d1d8ce3735 + + ---> 7b4d0911ff76 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 675698d063d5 + + ---> 1a18d54baf74 + +Step 12/12 : workdir /root + + + ---> Running in 2ed19cf8d53f + + ---> 72d333f861ca + +Successfully built 72d333f861ca + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e3cb14 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e9d826/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e9d826/docker-build-logs.txt new file mode 100644 index 00000000..e0d1a63c --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e9d826/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> ed363dd4c45a + +Step 10/12 : arg tag + + + ---> Running in 8224340eed49 + + ---> 7615be778d2e + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in b5ab54c2895a + + ---> 95a3e1a00804 + +Step 12/12 : workdir /root + + + ---> Running in 38b6e8c1bd26 + + ---> 46519672244d + +Successfully built 46519672244d + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-e9d826 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ebd0e4/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ebd0e4/docker-build-logs.txt new file mode 100644 index 00000000..75484f51 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ebd0e4/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 667dcc48b428 + +Step 10/12 : arg tag + + + ---> Running in b3354a35275a + + ---> bad87e668109 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in db82dda85426 + + ---> fbcc83f767f9 + +Step 12/12 : workdir /root + + + ---> Running in e59e8f4d4a98 + + ---> fd061e695587 + +Successfully built fd061e695587 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ebd0e4 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ecbaf0/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ecbaf0/docker-build-logs.txt new file mode 100644 index 00000000..704a8b1c --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ecbaf0/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 9dcf2cb19fde + +Step 10/12 : arg tag + + + ---> Running in e66916bdeda8 + + ---> 05e249ff80fa + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 083c1bb91e55 + + ---> f187b7a7cc14 + +Step 12/12 : workdir /root + + + ---> Running in 5c432580db84 + + ---> dbb5da47cf1c + +Successfully built dbb5da47cf1c + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-ecbaf0 + diff --git a/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-f07047/docker-build-logs.txt b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-f07047/docker-build-logs.txt new file mode 100644 index 00000000..83923260 --- /dev/null +++ b/.latch/.logs/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-f07047/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 3964add1db13 + +Step 10/12 : arg tag + + + ---> Running in 536cd810a243 + + ---> 4027320badd1 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in cf5b952816ef + + ---> ae6a0e53da54 + +Step 12/12 : workdir /root + + + ---> Running in 07e5b74335fc + + ---> c9f4f04759c3 + +Successfully built c9f4f04759c3 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35729_nf_nf_core_cutandrun:0.1.0-96a42f-wip-f07047 + diff --git a/.latch/.logs/35929_nf_nf_core_cutandrun:0.1.0-96a42f-wip-da774b/docker-build-logs.txt b/.latch/.logs/35929_nf_nf_core_cutandrun:0.1.0-96a42f-wip-da774b/docker-build-logs.txt new file mode 100644 index 00000000..3bfdbcda --- /dev/null +++ b/.latch/.logs/35929_nf_nf_core_cutandrun:0.1.0-96a42f-wip-da774b/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> a73027c4df84 + +Step 10/12 : arg tag + + + ---> Running in 3d21c680e2d8 + + ---> 77ee5fed91c2 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in 881bcf04697e + + ---> b31226e90f38 + +Step 12/12 : workdir /root + + + ---> Running in 24e7738bf159 + + ---> e222219fed2d + +Successfully built e222219fed2d + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/35929_nf_nf_core_cutandrun:0.1.0-96a42f-wip-da774b + diff --git a/.latch/.logs/36340_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d6e2a4/docker-build-logs.txt b/.latch/.logs/36340_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d6e2a4/docker-build-logs.txt new file mode 100644 index 00000000..95520f1d --- /dev/null +++ b/.latch/.logs/36340_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d6e2a4/docker-build-logs.txt @@ -0,0 +1,84 @@ +Step 1/12 : from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 + + + ---> 98d9cd0f2a3a + +Step 2/12 : workdir /tmp/docker-build/work/ + + + ---> Using cache + + ---> 8f95defb9629 + +Step 3/12 : shell [ "/usr/bin/env", "bash", "-o", "errexit", "-o", "pipefail", "-o", "nounset", "-o", "verbose", "-o", "errtrace", "-O", "inherit_errexit", "-O", "shift_verbose", "-c" ] + + + ---> Using cache + + ---> b2b2f8c4e3df + +Step 4/12 : env TZ='Etc/UTC' + + + ---> Using cache + + ---> ee07cb31686a + +Step 5/12 : env LANG='en_US.UTF-8' + + + ---> Using cache + + ---> b133cffabb9a + +Step 6/12 : arg DEBIAN_FRONTEND=noninteractive + + + ---> Using cache + + ---> 7f1ec8bb956e + +Step 7/12 : run pip install latch==2.52.1 + + + ---> Using cache + + ---> f6fd6659b394 + +Step 8/12 : run mkdir /opt/latch + + + ---> Using cache + + ---> 48d9448c23ba + +Step 9/12 : copy . /root/ + + + ---> 16fcabb41c1c + +Step 10/12 : arg tag + + + ---> Running in a303f7429f36 + + ---> 049d0a9255e6 + +Step 11/12 : env FLYTE_INTERNAL_IMAGE $tag + + + ---> Running in ecd02faaa3de + + ---> d489b7c09ede + +Step 12/12 : workdir /root + + + ---> Running in 5565daeeb7cc + + ---> 40bda76ee715 + +Successfully built 40bda76ee715 + +Successfully tagged 812206152185.dkr.ecr.us-west-2.amazonaws.com/36340_nf_nf_core_cutandrun:0.1.0-96a42f-wip-d6e2a4 + diff --git a/Dockerfile b/.latch/Dockerfile similarity index 65% rename from Dockerfile rename to .latch/Dockerfile index 3fba858e..0f308532 100644 --- a/Dockerfile +++ b/.latch/Dockerfile @@ -1,5 +1,5 @@ # DO NOT CHANGE -from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main +from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 workdir /tmp/docker-build/work/ @@ -21,21 +21,14 @@ arg DEBIAN_FRONTEND=noninteractive # Latch SDK # DO NOT REMOVE -run pip install latch==2.46.6 +run pip install latch==2.52.1 run mkdir /opt/latch -run apt-get update && apt-get install -y default-jre-headless # Copy workflow data (use .dockerignore to skip files) copy . /root/ -# Latch nextflow workflow entrypoint -# DO NOT CHANGE - -run ln -s /root/.latch/bin/nextflow /root/nextflow -run ln -s /root/.latch/.nextflow /root/.nextflow - # Latch workflow registration metadata # DO NOT CHANGE diff --git a/.latch/bin/nextflow b/.latch/bin/nextflow deleted file mode 100755 index 7906c2aa..00000000 --- a/.latch/bin/nextflow +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2013-2023, Seqera Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'23.11.0-edge'} -NXF_ORG=${NXF_ORG:-'nextflow-io'} -NXF_HOME=${NXF_HOME:-$HOME/.nextflow} -NXF_PROT=${NXF_PROT:-'https'} -NXF_BASE=${NXF_BASE:-$NXF_PROT://www.nextflow.io/releases} -NXF_TEMP=${NXF_TEMP:-$TMPDIR} -NXF_DIST=${NXF_DIST:-$NXF_HOME/framework} -NXF_CLI="$0 $@" -NXF_CLI_OPTS=${NXF_CLI_OPTS:-} -NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} - -export NXF_CLI -export NXF_ORG -export NXF_HOME - -if [[ $TERM && $TERM != 'dumb' ]]; then -if command -v tput &>/dev/null; then -GREEN=$(tput setaf 2; tput bold) -YELLOW=$(tput setaf 3) -RED=$(tput setaf 1) -NORMAL=$(tput sgr0) -fi -fi - -function echo_red() { - >&2 echo -e "$RED$*$NORMAL" -} - -function echo_green() { - echo -e "$GREEN$*$NORMAL" -} - -function echo_yellow() { - >&2 echo -e "$YELLOW$*$NORMAL" -} - -function die() { - echo_red "$*" - exit 1 -} - -function get_abs_filename() { - echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" -} - -function get() { - if command -v curl &>/dev/null; then - GET="curl -fsSL '$1' -o '$2'" - elif command -v wget &>/dev/null; then - GET="wget '$1' -O '$2' >/dev/null 2>&1" - else - echo_red "ERROR: Cannot find 'curl' nor 'wget' utility -- please install one of them" - exit 1 - fi - - printf "Downloading nextflow dependencies. It may require a few seconds, please wait .. " - eval $GET; status=$? - printf "\r\033[K" - if [ $status -ne 0 ]; then - echo_red "ERROR: Cannot download nextflow required file -- make sure you can connect to the internet" - echo "" - echo "Alternatively you can try to download this file:" - echo " $1" - echo "" - echo "and save it as:" - echo " ${3:-$2}" - echo "" - exit 1 - fi -} - -function get_ver() { - if command -v curl &>/dev/null; then - curl -fsSL "$1" - elif command -v wget &>/dev/null; then - wget "$1" >/dev/null 2>&1 - else - echo_red "ERROR: Cannot find 'curl' nor 'wget' utility -- please install one of them" - exit 1 - fi -} - -function make_temp() { - local base=${NXF_TEMP:=$PWD} - if [ "$(uname)" = 'Darwin' ]; then mktemp "${base}/nxf-tmp.XXXXXX" || exit $? - else mktemp -t nxf-tmp.XXXXXX -p "${base}" || exit $? - fi -} - -function resolve_link() { - [[ ! -f $1 ]] && exit 1 - if command -v realpath &>/dev/null; then - realpath "$1" - elif command -v readlink &>/dev/null; then - local target="$1" - cd "$(dirname "$target")"; target="$(basename "$target")" - while [ -L "$target" ]; do - target="$(readlink "$target")" - cd "$(dirname "$target")"; target="$(basename "$target")" - done - echo "$(cd "$(dirname "$target")"; pwd -P)/$target" - else - echo_yellow "WARN: Neither \`realpath\` nor \`readlink\` command can be found" - exit 1 - fi -} - -function current_ver() { - [[ $NXF_EDGE == 1 || $NXF_VER == *"-edge" ]] && printf 'edge' || printf 'latest' -} - -function install() { - local tmpfile=$(make_temp) - local version=$(set +u; [[ $NXF_VER ]] && printf "v$NXF_VER" || current_ver) - local action="a=${2:-default}" - get "$NXF_BASE/$version/nextflow?$action" "$tmpfile" "$1" || exit $? - mv "$tmpfile" "$1" || exit $? - chmod +x "$1" || exit $? - bash "$1" -download || exit $? - echo '' - echo -e $'Nextflow installation completed. Please note:' - echo -e $'- the executable file `nextflow` has been created in the folder:' $(dirname $1) - if [[ ! "$PATH" =~ (^|:)"$(dirname $1)"(:|$) ]]; then - echo -e $'- you may complete the installation by moving it to a directory in your $PATH' - fi - echo '' -} - -function check_latest() { - [[ $cmd != run ]] && return 0 - [[ $NXF_OFFLINE == true || $NXF_DISABLE_CHECK_LATEST == true ]] && return 0 - local latest=$(get_ver "$NXF_BASE/$(current_ver)/version?current=$NXF_VER") - if [[ -n "$latest" && "$latest" != $NXF_VER ]]; then - echo_yellow "Nextflow $latest is available - Please consider updating your version to it" - fi -} - -function launch_nextflow() { - # the launch command line - local cmdline=() - # remove leading and trailing double-quotes - for x in "${launcher[@]}"; do - x="${x%\"}" - x="${x#\"}" - cmdline+=("$x") - done - - if [[ "$bg" ]]; then - local pid_file="${NXF_PID_FILE:-.nextflow.pid}" - cmdline+=("${args[@]}") - exec "${cmdline[@]}" & - disown - echo $! > "$pid_file" - exit 0 - fi - - cmdline+=("${args[@]}") - exec "${cmdline[@]}" - exit 1 -} - -# check self-install -if [ "$0" = "bash" ] || [[ "$0" =~ .*/bash ]]; then - if [ -d nextflow ]; then - echo 'Please note:' - echo "- The install procedure needs to create a file named 'nextflow' in this folder, but a directory with this name already exists." - echo "- Please renamed/delete that directory, or execute the Nextflow install procedure in another folder." - echo '' - exit 1 - fi - install "$PWD/nextflow" install - exit 0 -fi - -# clean up env -# see https://github.com/nextflow-io/nextflow/issues/1716 -unset JAVA_TOOL_OPTIONS - -# parse the command line -bg='' -declare -a jvmopts=() -declare -a args=("$@") -declare -a commands=(clone config drop help history info ls pull run view node console kuberun) -# $NXF_CLI_OPTS allow to pass arbitrary cli opts via the environment -# note: do not wrap with quotes because the variable can be used to specify more than on option separate by blanks -[ "$NXF_CLI_OPTS" ] && args+=($NXF_CLI_OPTS) - -cmd='' -while [[ $# != 0 ]]; do - case $1 in - -D*) - if [[ ! "$cmd" ]]; then - jvmopts+=("$1") - fi - ;; - -bg) - bg=1 - ;; - -remote-debug) - echo_yellow "Enabling script debugging - continue the execution launching the remote VM debugger in your favourite IDE using port $NXF_REMOTE_DEBUG_PORT" - remote_debug=1 - ;; - -download) - if [[ ! "$cmd" ]]; then - rm -rf "$NXF_DIST/$NXF_VER" || exit $? - bash "$0" -version || exit $? - exit 0 - fi - ;; - -self-update|self-update) - if [[ ! "$cmd" ]]; then - [[ -z $NXF_EDGE && $NXF_VER = *-edge ]] && NXF_EDGE=1 - unset NXF_VER - install "$0" update - exit 0 - fi - ;; - *) - [[ $1 && $1 != -* && ! "$cmd" && ${commands[*]} =~ $1 ]] && cmd=$1 - ;; - esac - shift -done - -CAPSULE_LOG=${CAPSULE_LOG:=''} -CAPSULE_RESET=${CAPSULE_RESET:=''} -CAPSULE_CACHE_DIR=${CAPSULE_CACHE_DIR:="$NXF_HOME/capsule"} - -NXF_PACK=one -NXF_MODE=${NXF_MODE:-''} -NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar} -NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR} -NXF_PATH=$(dirname "$NXF_BIN") -NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR} -NXF_GRAB=${NXF_GRAB:-''} -NXF_CLASSPATH=${NXF_CLASSPATH:-''} -NXF_HOST=${HOSTNAME:-localhost} -[[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST} - -# Determine the path to this file -if [[ $NXF_PACK = all ]]; then - NXF_BIN=$(which "$0" 2>/dev/null) - [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" -fi - -# use nextflow custom java home path -if [[ "$NXF_JAVA_HOME" ]]; then - JAVA_HOME="$NXF_JAVA_HOME" - unset JAVA_CMD -fi -# Determine the Java command to use to start the JVM. -if [ ! -x "$JAVA_CMD" ] ; then - if [ -d "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVA_CMD="$JAVA_HOME/jre/sh/java" - else - JAVA_CMD="$JAVA_HOME/bin/java" - fi - elif [ -x /usr/libexec/java_home ]; then - JAVA_CMD="$(/usr/libexec/java_home -v 1.8+)/bin/java" - else - JAVA_CMD="$(which java)" || JAVA_CMD=java - fi -fi - -# Retrieve the java version from a NF local file -JAVA_KEY="$NXF_HOME/tmp/ver/$(resolve_link "$JAVA_CMD" | sed 's@/@.@g')" -if [ -f "$JAVA_KEY" ]; then - JAVA_VER="$(cat "$JAVA_KEY")" -else - JAVA_VER="$("$JAVA_CMD" $NXF_OPTS -version 2>&1)" - if [ $? -ne 0 ]; then - echo_red "${JAVA_VER:-Failed to launch the Java virtual machine}" - echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n NXF_OPTS: $NXF_OPTS\n" - exit 1 - fi - JAVA_VER=$(echo "$JAVA_VER" | awk '/version/ {gsub(/"/, "", $3); print $3}') - # check NF version - if [[ ! $NXF_VER =~ ([0-9]+)\.([0-9]+)\.([0-9].*) ]]; then - echo_red "Not a valid Nextflow version: $NXF_VER" - exit 1 - fi - major=${BASH_REMATCH[1]} - minor=${BASH_REMATCH[2]} - # legacy version - Java 7/8 only - if [ $major -eq 0 ] && [ $minor -lt 26 ]; then - version_check="^(1.7|1.8)" - version_message="Java 7 or 8" - else - version_check="^(1.8|9|10|11|12|13|14|15|16|17|18|19|20|21)" - version_message="Java 8 or later (up to 21)" - fi - if [[ ! $JAVA_VER =~ $version_check ]]; then - echo_red "ERROR: Cannot find Java or it's a wrong version -- please make sure that $version_message is installed" - if [[ "$NXF_JAVA_HOME" ]]; then - echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n NXF_JAVA_HOME: $NXF_JAVA_HOME\n" - else - echo_yellow "NOTE: Nextflow is trying to use the Java VM defined by the following environment variables:\n JAVA_CMD: $JAVA_CMD\n JAVA_HOME: $JAVA_HOME\n" - fi - exit 1 - fi - if [[ ! $JAVA_VER =~ ^(11|12|13|14|15|16|17|18|19|20|21) ]]; then - echo_yellow "NOTE: Nextflow is not tested with Java $JAVA_VER -- It's recommended the use of version 11 up to 21\n" - fi - mkdir -p "$(dirname "$JAVA_KEY")" - [[ -f $JAVA_VER ]] && echo $JAVA_VER > "$JAVA_KEY" -fi - -# Verify nextflow jar is available -if [ ! -f "$NXF_BIN" ]; then - [ -f "$NXF_PATH" ] && rm "$NXF_PATH" - mkdir -p "$NXF_PATH" || exit $? - tmpfile=$(make_temp) - get "$NXF_URL" "$tmpfile" "$NXF_BIN" - mv "$tmpfile" "$NXF_BIN" -fi - -COLUMNS=${COLUMNS:-`tty -s && tput cols 2>/dev/null || true`} -declare -a JAVA_OPTS=() -JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcapsule.trampoline -Dcapsule.java.cmd="$JAVA_CMD" -Dcom.sun.security.enableAIAcaIssuers=true) -if [[ $cmd == console ]]; then bg=1; -else JAVA_OPTS+=(-Djava.awt.headless=true) -fi - -[[ "$JAVA_VER" =~ ^(21) ]] && [[ ! "$NXF_ENABLE_VIRTUAL_THREADS" ]] && NXF_ENABLE_VIRTUAL_THREADS=true -[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") -[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) -[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) -[[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1) -[[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS) -[[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH -[[ "$NXF_GRAB" ]] && export NXF_GRAB -[[ "$COLUMNS" ]] && export COLUMNS -[[ "$NXF_TEMP" ]] && JAVA_OPTS+=(-Djava.io.tmpdir="$NXF_TEMP") -[[ "${jvmopts[@]}" ]] && JAVA_OPTS+=("${jvmopts[@]}") -export JAVA_CMD -export CAPSULE_CACHE_DIR -export NXF_PLUGINS_DIR -export NXF_PLUGINS_MODE -export NXF_PLUGINS_DEFAULT -export NXF_PACK -export NXF_ENABLE_VIRTUAL_THREADS - -# lookup the a `md5` command -if hash md5sum 2>/dev/null; then MD5=md5sum; -elif hash gmd5sum 2>/dev/null; then MD5=gmd5sum; -elif hash md5 2>/dev/null; then MD5=md5; -else MD5='' -fi - -# when no md5 command is available fallback on default execution -if [ ! "$MD5" ] || [ "$CAPSULE_RESET" ]; then - launcher=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) - launch_nextflow - exit 1 -fi - -# creates a md5 unique for the given variables -env_md5() { -cat </dev/null; then - STR='' - for x in "${launcher[@]}"; do - [[ "$x" != "\"-Duser.dir=$PWD\"" ]] && [[ ! "$x" == *"-agentlib:jdwp"* ]] && STR+=$(printf '%q ' "$x") - done - printf "$STR">"$LAUNCH_FILE" - else - echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?" - fi - -fi - -# check for latest version -check_latest -# finally run it -launch_nextflow diff --git a/.latch/config b/.latch/config index a8e1f41d..96f56f3f 100644 --- a/.latch/config +++ b/.latch/config @@ -1 +1 @@ -{"latch_version": "2.46.6", "base_image": "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:fe0b-main", "date": "2024-06-17T18:20:03.014880"} \ No newline at end of file +{"latch_version": "2.52.1", "base_image": "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0", "date": "2024-09-12T12:57:15.404807"} diff --git a/.latch/nextflow_version b/.latch/nextflow_version deleted file mode 100644 index c1757a8d..00000000 --- a/.latch/nextflow_version +++ /dev/null @@ -1 +0,0 @@ -v1.0.5 \ No newline at end of file diff --git a/.latch/workflow_name b/.latch/workflow_name index 3f929e9f..5a6730b4 100644 --- a/.latch/workflow_name +++ b/.latch/workflow_name @@ -1 +1 @@ -wf.entrypoint.nf_nf_core_cutandrun \ No newline at end of file +nf_nf_core_cutandrun \ No newline at end of file diff --git a/latch_metadata/__init__.py b/latch_metadata/__init__.py index d2738853..c6c0f146 100644 --- a/latch_metadata/__init__.py +++ b/latch_metadata/__init__.py @@ -1,17 +1,168 @@ - +from latch.ldata.path import LPath +from latch.types import metadata +from latch.types.directory import LatchDir, LatchOutputDir +from latch.types.file import LatchFile from latch.types.metadata import ( - NextflowMetadata, + Fork, + ForkBranch, LatchAuthor, - NextflowRuntimeResources + NextflowMetadata, + NextflowParameter, + NextflowRuntimeResources, + Params, + Section, + Spoiler, + Text, ) -from latch.types.directory import LatchDir from .parameters import generated_parameters +flow = [ + Section("Input/Output", Params("input", "outdir", "run_name"), Spoiler("Optional", Params("use_control"))), + Section( + "Reference Genome Options", + Fork( + "genome_source", + "", + latch_genome_source=ForkBranch("Latch Certified Reference Genome", Params("latch_genome")), + input_ref=ForkBranch( + "Custom Reference Genome", + Params("fasta", "gtf"), + Spoiler( + "Reference Options", + Params("gene_bed", "save_reference"), + Text( + "The transcriptome and GTF files in iGenomes are vastly out of date with respect to current annotations from Ensembl e.g. human iGenomes annotations are from Ensembl release 75, while the current Ensembl release is 108. Please consider downloading and using a more updated version of your reference genome." + ), + Params("genome"), + ), + ), + ), + ), + Spoiler( + "Spikein Genome Options", + Params( + "spikein_genome", + "spikein_bowtie2", + "spikein_fasta", + "save_spikein_aligned", + ), + ), + Spoiler( + "Optional Parameters", + Spoiler( + "Alignment Filters", + Params( + "bowtie2", + "blacklist", + "remove_mitochondrial_reads", + "mito_name", + "minimum_alignment_q_score", + "save_unaligned", + "save_align_intermed", + "remove_linear_duplicates", + "end_to_end", + ), + ), + Spoiler( + "Peak Calling Options", + Params("peakcaller"), + Spoiler( + "MACS2 Optional Parameters", + Params( + "macs2_qvalue", + "macs2_pvalue", + "macs_gsize", + "macs2_narrow_peak", + "macs2_broad_cutoff", + "consensus_peak_mode", + "replicate_threshold", + "extend_fragments", + ), + ), + Spoiler( + "SEACR Options", + Params( + "seacr_peak_threshold", + "seacr_norm", + "seacr_stringent", + ), + ), + ), + Spoiler( + "Read trimming and QC Options", + Params( + "save_merged_fastq", + "save_trimmed", + "clip_r1", + "clip_r2", + "three_prime_clip_r1", + "three_prime_clip_r2", + "trim_nextseq", + "dedup_target_reads", + ), + ), + Spoiler( + "Normalization Options", + Params("normalisation_mode", "normalisation_binsize", "igg_scale_factor", "dump_scale_factors"), + ), + Spoiler( + "Reporting/Plotting Options", + Params( + "dt_qc_bam_binsize", + "dt_qc_corr_method", + "dt_calc_all_matrix", + "dt_heatmap_gene_beforelen", + "dt_heatmap_gene_bodylen", + "dt_heatmap_gene_afterlen", + "dt_heatmap_peak_beforelen", + "dt_heatmap_peak_afterlen", + "min_frip_overlap", + "min_peak_overlap", + "igv_show_gene_names", + "igv_sort_by_groups", + ), + ), + Spoiler( + "Flow Switching Options", + Params( + "only_input", + "only_genome", + "only_preqc", + "only_alignment", + "only_filtering", + "only_peak_calling", + ), + ), + Spoiler( + "Skip Tools", + Params( + "skip_fastqc", + "skip_trimming", + "skip_removeduplicates", + "skip_reporting", + "skip_preseq", + "skip_igv", + "skip_dt_qc", + "skip_heatmaps", + "skip_peak_qc", + "skip_multiqc", + ), + ), + Spoiler( + "MultiQC Options", + Params( + "multiqc_title", + "multiqc_methods_description", + ), + ), + ), +] + NextflowMetadata( - display_name='nf-core/cutandrun', + display_name="nf-core/cutandrun", author=LatchAuthor( - name="Your Name", + name="nf-core", ), parameters=generated_parameters, runtime_resources=NextflowRuntimeResources( @@ -20,4 +171,5 @@ storage_gib=100, ), log_dir=LatchDir("latch:///your_log_dir"), + flow=flow, ) diff --git a/latch_metadata/parameters.py b/latch_metadata/parameters.py index da8993ed..c2a58281 100644 --- a/latch_metadata/parameters.py +++ b/latch_metadata/parameters.py @@ -1,480 +1,596 @@ - -from dataclasses import dataclass +import csv import typing -import typing_extensions +from dataclasses import dataclass +from enum import Enum +import typing_extensions from flytekit.core.annotation import FlyteAnnotation - -from latch.types.metadata import NextflowParameter -from latch.types.file import LatchFile from latch.types.directory import LatchDir, LatchOutputDir +from latch.types.file import LatchFile +from latch.types.metadata import Multiselect, NextflowParameter # Import these into your `__init__.py` file: # # from .parameters import generated_parameters + +@dataclass +class SampleSheet: + group: str + replicate: int + fastq_1: LatchFile + fastq_2: LatchFile + control: str + + +class Reference(Enum): + hg19 = "GRCh37 (Homo Sapiens hg19)" + hg38 = "GRCh38 (Homo Sapiens hg38)" + + generated_parameters = { - 'input': NextflowParameter( - type=LatchFile, + "input": NextflowParameter( + type=typing.List[SampleSheet], + display_name="Input", + description="Information about the samples in the experiment", + section_title=None, + samplesheet_type="csv", + samplesheet=True, default=None, - section_title='Input/output options', - description='Path to comma-separated file containing information about the samples in the experiment.', ), - 'outdir': NextflowParameter( - type=typing_extensions.Annotated[LatchDir, FlyteAnnotation({'output': True})], + "outdir": NextflowParameter( + type=LatchOutputDir, # typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], default=None, - section_title=None, - description='The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.', + display_name="Outdir", + description="The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.", ), - 'multiqc_title': NextflowParameter( + "multiqc_title": NextflowParameter( type=typing.Optional[str], default=None, section_title=None, - description='MultiQC report title. Printed as page header, used for filename if not otherwise specified.', + display_name="MultiQC Title", + description="MultiQC report title. Printed as page header, used for filename if not otherwise specified.", ), - 'save_reference': NextflowParameter( - type=typing.Optional[bool], + "save_reference": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save genome reference data to the output directory', + display_name="Save Reference", + description="Save genome reference data to the output directory", ), - 'save_merged_fastq': NextflowParameter( - type=typing.Optional[bool], + "save_merged_fastq": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save any technical replicate FASTQ files that were merged to the output directory', + display_name="Save Merged FastQ", + description="Save any technical replicate FASTQ files that were merged to the output directory", ), - 'save_trimmed': NextflowParameter( - type=typing.Optional[bool], + "genome_source": NextflowParameter( + type=str, + display_name="Reference Genome", + description="Choose Reference Genome", + ), + "latch_genome": NextflowParameter( + type=typing.Optional[Reference], + default=None, + section_title="Reference genome options", + display_name="Genome", + description="Name of the Latch Verfied Reference Genome", + ), + "save_trimmed": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save trimmed FASTQ files to the output directory', + display_name="Save Trimmed", + description="Save trimmed FASTQ files to the output directory", ), - 'save_spikein_aligned': NextflowParameter( - type=typing.Optional[bool], + "save_spikein_aligned": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save BAM files aligned to the spike-in genome to the output directory', + display_name="Save Spikein Aligned", + description="Save BAM files aligned to the spike-in genome to the output directory", ), - 'save_unaligned': NextflowParameter( - type=typing.Optional[bool], + "save_unaligned": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save unaligned sequences to the output directory', + display_name="Save Unaligned", + description="Save unaligned sequences to the output directory", ), - 'save_align_intermed': NextflowParameter( - type=typing.Optional[bool], + "save_align_intermed": NextflowParameter( + type=bool, default=None, section_title=None, - description='Save alignment intermediates to the output directory (WARNING: can be very large)', + display_name="Save Align Intermed", + description="Save alignment intermediates to the output directory (WARNING: can be very large)", ), - 'email': NextflowParameter( + "email": NextflowParameter( type=typing.Optional[str], default=None, section_title=None, - description='Email address for completion summary.', + display_name="Email", + description="Email address for completion summary.", ), - 'dump_scale_factors': NextflowParameter( - type=typing.Optional[bool], + "dump_scale_factors": NextflowParameter( + type=bool, default=None, section_title=None, - description='Output calculated scale factors from pipeline', + display_name="Dump Scale Factors", + description="Output calculated scale factors from pipeline", ), - 'genome': NextflowParameter( + "genome": NextflowParameter( type=typing.Optional[str], default=None, - section_title='Reference data options', - description='Name of iGenomes reference.', + section_title=None, + display_name="Genome", + description="Name of iGenomes reference.", ), - 'bowtie2': NextflowParameter( - type=typing.Optional[str], + "bowtie2": NextflowParameter( + type=typing.Optional[LatchDir], default=None, section_title=None, - description='Path to bowtie2 index', + display_name="Bowtie2", + description="Path to bowtie2 index", ), - 'gtf': NextflowParameter( - type=typing.Optional[str], + "gtf": NextflowParameter( + type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to GTF annotation file', + display_name="GTF", + description="Path to GTF annotation file", ), - 'gene_bed': NextflowParameter( - type=typing.Optional[str], + "gene_bed": NextflowParameter( + type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to gene BED file', + display_name="Gene Bed", + description="Path to gene BED file", ), - 'blacklist': NextflowParameter( - type=typing.Optional[str], + "blacklist": NextflowParameter( + type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to genome blacklist', + display_name="Blacklist", + description="Path to genome blacklist", ), - 'spikein_genome': NextflowParameter( + "spikein_genome": NextflowParameter( type=typing.Optional[str], - default='K12-MG1655', + default="K12-MG1655", section_title=None, - description='Name of the iGenome reference for the spike-in genome, defaulting to E. coli K12, for yeast set to R64-1-1, for fruit fly BDGP6', + display_name="Spikein Genome", + description="Name of the iGenome reference for the spike-in genome, defaulting to E. coli K12, for yeast set to R64-1-1, for fruit fly BDGP6", ), - 'spikein_bowtie2': NextflowParameter( - type=typing.Optional[str], + "spikein_bowtie2": NextflowParameter( + type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to spike-in bowtie2 index', + display_name="Spikein Bowtie2", + description="Path to spike-in bowtie2 index", ), - 'spikein_fasta': NextflowParameter( - type=typing.Optional[str], + "spikein_fasta": NextflowParameter( + type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to spike-in fasta', + display_name="Spikein Fasta", + description="Path to spike-in fasta", ), - 'fasta': NextflowParameter( + "fasta": NextflowParameter( type=typing.Optional[LatchFile], default=None, section_title=None, - description='Path to FASTA genome file.', + display_name="Fasta", + description="Path to FASTA genome file.", ), - 'only_input': NextflowParameter( - type=typing.Optional[bool], + "only_input": NextflowParameter( + type=bool, default=None, - section_title='Flow switching options', - description='Run pipeline up to input checking', + section_title=None, + display_name="Only Input", + description="Run pipeline up to input checking", ), - 'only_genome': NextflowParameter( - type=typing.Optional[bool], + "only_genome": NextflowParameter( + type=bool, default=None, section_title=None, - description='Run pipeline up to reference preparation', + display_name="Only Genome", + description="Run pipeline up to reference preparation", ), - 'only_preqc': NextflowParameter( - type=typing.Optional[bool], + "only_preqc": NextflowParameter( + type=bool, default=None, section_title=None, - description='Run pipeline up to pre-alignment', + display_name="Only PreQC", + description="Run pipeline up to pre-alignment", ), - 'only_alignment': NextflowParameter( - type=typing.Optional[bool], + "only_alignment": NextflowParameter( + type=bool, default=None, section_title=None, - description='Run pipeline up to alignment', + display_name="Only Alignment", + description="Run pipeline up to alignment", ), - 'only_filtering': NextflowParameter( - type=typing.Optional[bool], + "only_filtering": NextflowParameter( + type=bool, default=None, section_title=None, - description='Run pipeline up to q-filtering', + display_name="Only Filtering", + description="Run pipeline up to q-filtering", ), - 'only_peak_calling': NextflowParameter( - type=typing.Optional[bool], + "only_peak_calling": NextflowParameter( + type=bool, default=None, section_title=None, - description='Run pipeline up to peak calling', + display_name="Only Peak Calling", + description="Run pipeline up to peak calling", ), - 'skip_fastqc': NextflowParameter( - type=typing.Optional[bool], + "skip_fastqc": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips fastqc reporting', + display_name="Skip FastQC", + description="Skips fastqc reporting", ), - 'skip_trimming': NextflowParameter( - type=typing.Optional[bool], + "skip_trimming": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips trimming', + display_name="Skip Trimming", + description="Skips trimming", ), - 'skip_removeduplicates': NextflowParameter( - type=typing.Optional[bool], + "skip_removeduplicates": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips de-duplication', + display_name="Skip Remove Duplicates", + description="Skips de-duplication", ), - 'skip_reporting': NextflowParameter( - type=typing.Optional[bool], + "skip_reporting": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips reporting', + display_name="Skip Reporting", + description="Skips reporting", ), - 'skip_preseq': NextflowParameter( - type=typing.Optional[bool], + "skip_preseq": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips preseq reporting', + display_name="Skip Preseq", + description="Skips preseq reporting", ), - 'skip_igv': NextflowParameter( - type=typing.Optional[bool], + "skip_igv": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips igv session generation', + display_name="Skip IGV", + description="Skips igv session generation", ), - 'skip_dt_qc': NextflowParameter( - type=typing.Optional[bool], + "skip_dt_qc": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips deeptools QC repoting', + display_name="Skip DT QC", + description="Skips deeptools QC reporting", ), - 'skip_heatmaps': NextflowParameter( - type=typing.Optional[bool], + "skip_heatmaps": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips deeptools heatmap generation', + display_name="Skip Heatmaps", + description="Skips deeptools heatmap generation", ), - 'skip_peak_qc': NextflowParameter( - type=typing.Optional[bool], + "skip_peak_qc": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips peak QC reporting', + display_name="Skip Peak QC", + description="Skips peak QC reporting", ), - 'skip_multiqc': NextflowParameter( - type=typing.Optional[bool], + "skip_multiqc": NextflowParameter( + type=bool, default=None, section_title=None, - description='Skips multiqc', + display_name="Skip MultiQC", + description="Skips multiqc", ), - 'clip_r1': NextflowParameter( + "clip_r1": NextflowParameter( type=typing.Optional[int], default=0, - section_title='Trimming Options', + section_title=None, + display_name="Clip R1", description="Instructs Trim Galore to remove bp from the 5' end of read 1 (or single-end reads).", ), - 'clip_r2': NextflowParameter( + "clip_r2": NextflowParameter( type=typing.Optional[int], default=0, section_title=None, + display_name="Clip R2", description="Instructs Trim Galore to remove bp from the 5' end of read 2 (paired-end reads only).", ), - 'three_prime_clip_r1': NextflowParameter( + "three_prime_clip_r1": NextflowParameter( type=typing.Optional[int], default=0, section_title=None, + display_name="Three Prime Clip R1", description="Instructs Trim Galore to remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed.", ), - 'three_prime_clip_r2': NextflowParameter( + "three_prime_clip_r2": NextflowParameter( type=typing.Optional[int], default=0, section_title=None, + display_name="Three Prime Clip R2", description="Instructs Trim Galore to remove bp from the 3' end of read 2 AFTER adapter/quality trimming has been performed.", ), - 'trim_nextseq': NextflowParameter( + "trim_nextseq": NextflowParameter( type=typing.Optional[int], default=0, section_title=None, - description='Instructs Trim Galore to apply the --nextseq=X option, to trim based on quality after removing poly-G tails.', + display_name="Trim NextSeq", + description="Instructs Trim Galore to apply the --nextseq=X option, to trim based on quality after removing poly-G tails.", ), - 'minimum_alignment_q_score': NextflowParameter( + "minimum_alignment_q_score": NextflowParameter( type=typing.Optional[int], default=20, - section_title='Pipeline Options', - description='Filter reads below a q-score threshold', + section_title=None, + display_name="Minimum Alignment Q Score", + description="Filter reads below a q-score threshold", ), - 'remove_mitochondrial_reads': NextflowParameter( - type=typing.Optional[bool], + "remove_mitochondrial_reads": NextflowParameter( + type=bool, default=None, section_title=None, - description='Filter mitochondrial reads', + display_name="Remove Mitochondrial Reads", + description="Filter mitochondrial reads", ), - 'mito_name': NextflowParameter( + "run_name": NextflowParameter( + type=str, + default=None, + section_title=None, + display_name="Run Name", + description="Run Name", + ), + "mito_name": NextflowParameter( type=typing.Optional[str], default=None, section_title=None, - description='Name of mitochondrial reads in reference genome. Only necessary when using a custom (non-igenomes) reference genome.', + display_name="Mito Name", + description="Name of mitochondrial reads in reference genome. Only necessary when using a custom (non-igenomes) reference genome.", ), - 'dedup_target_reads': NextflowParameter( - type=typing.Optional[bool], + "dedup_target_reads": NextflowParameter( + type=bool, default=None, section_title=None, - description='De-duplicate target reads AND control reads (default is control only)', + display_name="Dedup Target Reads", + description="De-duplicate target reads AND control reads (default is control only)", ), - 'remove_linear_duplicates': NextflowParameter( - type=typing.Optional[bool], + "remove_linear_duplicates": NextflowParameter( + type=bool, default=None, section_title=None, + display_name="Remove Linear Duplicates", description="De-duplicate reads based on read 1 5' start position. Relevant for assays using linear amplification with tagmentation (default is false).", ), - 'end_to_end': NextflowParameter( - type=typing.Optional[bool], + "end_to_end": NextflowParameter( + type=bool, default=True, section_title=None, - description='Use --end-to-end mode of Bowtie2 during alignment', + display_name="End to End", + description="Use --end-to-end mode of Bowtie2 during alignment", ), - 'normalisation_mode': NextflowParameter( + "normalisation_mode": NextflowParameter( type=typing.Optional[str], - default='Spikein', + default="Spikein", section_title=None, + display_name="Normalisation Mode", + appearance_type=Multiselect(["Spikein", "RPKM", "CPM", "BPM", "None"], allow_custom=False), description='Sets the target read normalisation mode. Options are: ["Spikein", "RPKM", "CPM", "BPM", "None" ]', ), - 'normalisation_binsize': NextflowParameter( + "normalisation_binsize": NextflowParameter( type=typing.Optional[int], default=50, section_title=None, - description='If normsalisation option is one of "RPKM", "CPM", "BPM" - then the binsize that the reads count is calculated on is used.', + display_name="Normalisation Binsize", + description='If normalisation option is one of "RPKM", "CPM", "BPM" - then the binsize that the reads count is calculated on is used.', ), - 'peakcaller': NextflowParameter( + "peakcaller": NextflowParameter( type=typing.Optional[str], - default='seacr', + default="seacr", section_title=None, - description='Selects the peak caller for the pipeline. Options are: [seacr, macs2]. More than one peak caller can be chosen and the order specifies which is a primary peak called (the first) that will be used downstream. Any secondary peak callers will be run and outputed to the results folder.', + display_name="Peakcaller", + appearance_type=Multiselect(["seacr", "macs2"], allow_custom=False), + description="Selects the peak caller for the pipeline. Options are: [seacr, macs2]. More than one peak caller can be chosen and the order specifies which is a primary peak called (the first) that will be used downstream. Any secondary peak callers will be run and outputed to the results folder.", ), - 'use_control': NextflowParameter( - type=typing.Optional[bool], + "use_control": NextflowParameter( + type=bool, default=True, section_title=None, - description='Specifies whether to use a control to normalise peak calls against (e.g. IgG)', + display_name="Use Control", + description="Specifies whether to use a control to normalise peak calls against (e.g. IgG)", ), - 'extend_fragments': NextflowParameter( - type=typing.Optional[bool], + "extend_fragments": NextflowParameter( + type=bool, default=True, section_title=None, - description='Specifies whether to extend paired-end fragments between the read mates when calculating coveage tracks', + display_name="Extend Fragments", + description="Specifies whether to extend paired-end fragments between the read mates when calculating coverage tracks", ), - 'igg_scale_factor': NextflowParameter( + "igg_scale_factor": NextflowParameter( type=typing.Optional[float], default=0.5, section_title=None, - description='Specifies whether the background control is scaled prior to being used to normalise peaks.', + display_name="IGG Scale Factor", + description="Specifies whether the background control is scaled prior to being used to normalise peaks.", ), - 'seacr_peak_threshold': NextflowParameter( + "seacr_peak_threshold": NextflowParameter( type=typing.Optional[float], default=0.05, section_title=None, - description='SEACR specifies returns the top n fraction (between 0 and 1) of peaks based on total signal within peaks. This is only used if there are no controls included with the samples and if `--use_control` is `false`', + display_name="SEACR Peak Threshold", + description="SEACR specifies returns the top n fraction (between 0 and 1) of peaks based on total signal within peaks. This is only used if there are no controls included with the samples and if `--use_control` is `false`", ), - 'seacr_norm': NextflowParameter( + "seacr_norm": NextflowParameter( type=typing.Optional[str], - default='non', + default="non", section_title=None, - description='SEACR normalization. ', + appearance_type=Multiselect(["non", "norm"], allow_custom=False), + display_name="SEACR Norm", + description="SEACR normalization.", ), - 'seacr_stringent': NextflowParameter( + "seacr_stringent": NextflowParameter( type=typing.Optional[str], - default='stringent', + default="stringent", section_title=None, - description='SEACR stringency.', + appearance_type=Multiselect(["stringent", "relaxed"], allow_custom=False), + display_name="SEACR Stringency", + description="SEACR stringency.", ), - 'macs2_qvalue': NextflowParameter( + "macs2_qvalue": NextflowParameter( type=typing.Optional[float], default=0.01, section_title=None, - description='Q-value threshold for MACS2 peak caller.', + display_name="MACS2 Q-value", + description="Q-value threshold for MACS2 peak caller.", ), - 'macs2_pvalue': NextflowParameter( + "macs2_pvalue": NextflowParameter( type=typing.Optional[float], default=None, section_title=None, - description='P-value threshold for macs2 peak caller. If set it will overide the qvalue.', + display_name="MACS2 P-value", + description="P-value threshold for macs2 peak caller. If set it will override the q-value.", ), - 'macs_gsize': NextflowParameter( + "macs_gsize": NextflowParameter( type=typing.Optional[float], - default=2700000000, + default=2700000000.0, section_title=None, - description='parameter required by MACS2. If using an iGenomes reference these have been provided when `--genome` is set as *GRCh37*, *GRCh38*, *GRCm38*, *WBcel235*, *BDGP6*, *R64-1-1*, *EF2*, *hg38*, *hg19* and *mm10*. Otherwise the gsize will default to GRCh38.', + display_name="MACS Gsize", + description="Parameter required by MACS2. If using an iGenomes reference these have been provided when `--genome` is set as *GRCh37*, *GRCh38*, *GRCm38*, *WBcel235*, *BDGP6*, *R64-1-1*, *EF2*, *hg38*, *hg19* and *mm10*. Otherwise the gsize will default to GRCh38.", ), - 'macs2_narrow_peak': NextflowParameter( - type=typing.Optional[bool], + "macs2_narrow_peak": NextflowParameter( + type=bool, default=True, section_title=None, - description='Determines whether MACS2 broad or narrow peak mode is used for the peak caller', + display_name="MACS2 Narrow Peak", + description="Determines whether MACS2 broad or narrow peak mode is used for the peak caller", ), - 'macs2_broad_cutoff': NextflowParameter( + "macs2_broad_cutoff": NextflowParameter( type=typing.Optional[float], default=0.1, section_title=None, - description='MACS2 broad cutoff parameter', + display_name="MACS2 Broad Cutoff", + description="MACS2 broad cutoff parameter", ), - 'consensus_peak_mode': NextflowParameter( + "consensus_peak_mode": NextflowParameter( type=typing.Optional[str], - default='group', + default="group", section_title=None, - description='Specifies what samples to group together for consensus peaks. Options are [group, all]', + display_name="Consensus Peak Mode", + appearance_type=Multiselect(["group", "all"], allow_custom=False), + description="Specifies what samples to group together for consensus peaks. Options are [group, all]", ), - 'replicate_threshold': NextflowParameter( + "replicate_threshold": NextflowParameter( type=typing.Optional[float], - default=1, + default=1.0, section_title=None, - description='Minimum number of overlapping replicates needed for a consensus peak', + display_name="Replicate Threshold", + description="Minimum number of overlapping replicates needed for a consensus peak", ), - 'igv_show_gene_names': NextflowParameter( - type=typing.Optional[bool], + "igv_show_gene_names": NextflowParameter( + type=bool, default=True, section_title=None, - description='Show gene names instead of symbols in IGV browser sessions', + display_name="IGV Show Gene Names", + description="Show gene names instead of symbols in IGV browser sessions", ), - 'dt_qc_bam_binsize': NextflowParameter( + "dt_qc_bam_binsize": NextflowParameter( type=typing.Optional[int], default=500, - section_title='Reporting Options', - description='Deeptools multiBamSummary bam bin size', + section_title=None, + display_name="DT QC BAM Binsize", + description="Deeptools multiBamSummary bam bin size", ), - 'dt_qc_corr_method': NextflowParameter( + "dt_qc_corr_method": NextflowParameter( type=typing.Optional[str], - default='pearson', + default="pearson", section_title=None, - description='Deeptools Correlation Plot statistical calculation method', + display_name="DT QC Corr Method", + description="Deeptools Correlation Plot statistical calculation method", ), - 'dt_heatmap_gene_beforelen': NextflowParameter( + "dt_heatmap_gene_beforelen": NextflowParameter( type=typing.Optional[int], default=3000, section_title=None, - description='Deeptools heatmap gene plot before length (bases)', + display_name="DT Heatmap Gene Beforelen", + description="Deeptools heatmap gene plot before length (bases)", ), - 'dt_heatmap_gene_bodylen': NextflowParameter( + "dt_heatmap_gene_bodylen": NextflowParameter( type=typing.Optional[int], default=5000, section_title=None, - description='Deeptools heatmap gene plot body length (bases)', + display_name="DT Heatmap Gene Bodylen", + description="Deeptools heatmap gene plot body length (bases)", ), - 'dt_heatmap_gene_afterlen': NextflowParameter( + "dt_heatmap_gene_afterlen": NextflowParameter( type=typing.Optional[int], default=3000, section_title=None, - description='Deeptools heatmap gene plot after length (bases)', + display_name="DT Heatmap Gene Afterlen", + description="Deeptools heatmap gene plot after length (bases)", ), - 'dt_heatmap_peak_beforelen': NextflowParameter( + "dt_heatmap_peak_beforelen": NextflowParameter( type=typing.Optional[int], default=3000, section_title=None, - description='Deeptools heatmap peak plot before length (bases)', + display_name="DT Heatmap Peak Beforelen", + description="Deeptools heatmap peak plot before length (bases)", ), - 'dt_heatmap_peak_afterlen': NextflowParameter( + "dt_heatmap_peak_afterlen": NextflowParameter( type=typing.Optional[int], default=3000, section_title=None, - description='Deeptools heatmap peak plot after length (bases)', + display_name="DT Heatmap Peak Afterlen", + description="Deeptools heatmap peak plot after length (bases)", ), - 'dt_calc_all_matrix': NextflowParameter( - type=typing.Optional[bool], + "dt_calc_all_matrix": NextflowParameter( + type=bool, default=True, section_title=None, - description='Flag for whether to generate a heatmap for all samples together', + display_name="DT Calc All Matrix", + description="Flag for whether to generate a heatmap for all samples together", ), - 'min_frip_overlap': NextflowParameter( + "min_frip_overlap": NextflowParameter( type=typing.Optional[float], default=0.2, section_title=None, - description='Minimum fragment overlap for FriP score', + display_name="Min FRiP Overlap", + description="Minimum fragment overlap for FriP score", ), - 'min_peak_overlap': NextflowParameter( + "min_peak_overlap": NextflowParameter( type=typing.Optional[float], default=0.2, section_title=None, - description='Minimum peak overlap for peak reproducibility plot', + display_name="Min Peak Overlap", + description="Minimum peak overlap for peak reproducibility plot", ), - 'igv_sort_by_groups': NextflowParameter( - type=typing.Optional[bool], + "igv_sort_by_groups": NextflowParameter( + type=bool, default=True, section_title=None, - description='Sort the IGV output tracks by group', + display_name="IGV Sort by Groups", + description="Sort the IGV output tracks by group", ), - 'singularity_pull_docker_container': NextflowParameter( - type=typing.Optional[bool], + "singularity_pull_docker_container": NextflowParameter( + type=bool, default=None, - section_title='Generic options', - description='Pull Docker container.', + section_title=None, + display_name="Singularity Pull Docker Container", + description="Pull Docker container.", ), - 'multiqc_methods_description': NextflowParameter( + "multiqc_methods_description": NextflowParameter( type=typing.Optional[str], default=None, section_title=None, - description='Custom MultiQC yaml file containing HTML including a methods description.', + display_name="MultiQC Methods Description", + description="Custom MultiQC yaml file containing HTML including a methods description.", ), } - diff --git a/wf/__init__.py b/wf/__init__.py new file mode 100644 index 00000000..2d1e9244 --- /dev/null +++ b/wf/__init__.py @@ -0,0 +1,254 @@ +import typing + +import typing_extensions +from flytekit.core.annotation import FlyteAnnotation +from latch.executions import report_nextflow_used_storage +from latch.ldata.path import LPath +from latch.resources.launch_plan import LaunchPlan +from latch.resources.tasks import custom_task, nextflow_runtime_task +from latch.resources.workflow import workflow +from latch.types import metadata +from latch.types.directory import LatchDir, LatchOutputDir +from latch.types.file import LatchFile + +from wf.entrypoint import Reference, SampleSheet, initialize, nextflow_runtime + + +@workflow(metadata._nextflow_metadata) +def nf_nf_core_cutandrun( + input: typing.List[SampleSheet], + outdir: LatchOutputDir, + multiqc_title: typing.Optional[str], + save_reference: bool, + save_merged_fastq: bool, + save_trimmed: bool, + save_spikein_aligned: bool, + save_unaligned: bool, + save_align_intermed: bool, + email: typing.Optional[str], + dump_scale_factors: bool, + genome_source: str, + genome: typing.Optional[str], + latch_genome: typing.Optional[Reference], + bowtie2: typing.Optional[LatchDir], + gtf: typing.Optional[LatchFile], + gene_bed: typing.Optional[LatchFile], + blacklist: typing.Optional[LatchFile], + spikein_bowtie2: typing.Optional[LatchFile], + spikein_fasta: typing.Optional[LatchFile], + fasta: typing.Optional[LatchFile], + only_input: bool, + only_genome: bool, + only_preqc: bool, + only_alignment: bool, + only_filtering: bool, + only_peak_calling: bool, + skip_fastqc: bool, + skip_trimming: bool, + skip_removeduplicates: bool, + skip_reporting: bool, + skip_preseq: bool, + skip_igv: bool, + skip_dt_qc: bool, + skip_heatmaps: bool, + skip_peak_qc: bool, + skip_multiqc: bool, + remove_mitochondrial_reads: bool, + run_name: str, + mito_name: typing.Optional[str], + dedup_target_reads: bool, + remove_linear_duplicates: bool, + macs2_pvalue: typing.Optional[float], + singularity_pull_docker_container: bool, + multiqc_methods_description: typing.Optional[str], + spikein_genome: typing.Optional[str] = "K12-MG1655", + clip_r1: typing.Optional[int] = 0, + clip_r2: typing.Optional[int] = 0, + three_prime_clip_r1: typing.Optional[int] = 0, + three_prime_clip_r2: typing.Optional[int] = 0, + trim_nextseq: typing.Optional[int] = 0, + minimum_alignment_q_score: typing.Optional[int] = 20, + end_to_end: bool = True, + normalisation_mode: typing.Optional[str] = "Spikein", + normalisation_binsize: typing.Optional[int] = 50, + peakcaller: typing.Optional[str] = "seacr", + use_control: bool = True, + extend_fragments: bool = True, + igg_scale_factor: typing.Optional[float] = 0.5, + seacr_peak_threshold: typing.Optional[float] = 0.05, + seacr_norm: typing.Optional[str] = "non", + seacr_stringent: typing.Optional[str] = "stringent", + macs2_qvalue: typing.Optional[float] = 0.01, + macs_gsize: typing.Optional[float] = 2700000000.0, + macs2_narrow_peak: bool = True, + macs2_broad_cutoff: typing.Optional[float] = 0.1, + consensus_peak_mode: typing.Optional[str] = "group", + replicate_threshold: typing.Optional[float] = 1.0, + igv_show_gene_names: bool = True, + dt_qc_bam_binsize: typing.Optional[int] = 500, + dt_qc_corr_method: typing.Optional[str] = "pearson", + dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, + dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, + dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, + dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, + dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, + dt_calc_all_matrix: bool = True, + min_frip_overlap: typing.Optional[float] = 0.2, + min_peak_overlap: typing.Optional[float] = 0.2, + igv_sort_by_groups: bool = True, +) -> None: + """ + nf-core/cutandrun + + Sample Description + """ + + pvc_name: str = initialize() + nextflow_runtime( + pvc_name=pvc_name, + input=input, + outdir=outdir, + multiqc_title=multiqc_title, + save_reference=save_reference, + save_merged_fastq=save_merged_fastq, + save_trimmed=save_trimmed, + save_spikein_aligned=save_spikein_aligned, + save_unaligned=save_unaligned, + save_align_intermed=save_align_intermed, + email=email, + dump_scale_factors=dump_scale_factors, + genome=genome, + genome_source=genome_source, + latch_genome=latch_genome, + bowtie2=bowtie2, + gtf=gtf, + gene_bed=gene_bed, + blacklist=blacklist, + spikein_genome=spikein_genome, + spikein_bowtie2=spikein_bowtie2, + spikein_fasta=spikein_fasta, + fasta=fasta, + only_input=only_input, + only_genome=only_genome, + only_preqc=only_preqc, + only_alignment=only_alignment, + only_filtering=only_filtering, + only_peak_calling=only_peak_calling, + skip_fastqc=skip_fastqc, + skip_trimming=skip_trimming, + skip_removeduplicates=skip_removeduplicates, + skip_reporting=skip_reporting, + skip_preseq=skip_preseq, + skip_igv=skip_igv, + skip_dt_qc=skip_dt_qc, + skip_heatmaps=skip_heatmaps, + skip_peak_qc=skip_peak_qc, + skip_multiqc=skip_multiqc, + clip_r1=clip_r1, + clip_r2=clip_r2, + three_prime_clip_r1=three_prime_clip_r1, + three_prime_clip_r2=three_prime_clip_r2, + trim_nextseq=trim_nextseq, + minimum_alignment_q_score=minimum_alignment_q_score, + remove_mitochondrial_reads=remove_mitochondrial_reads, + run_name=run_name, + mito_name=mito_name, + dedup_target_reads=dedup_target_reads, + remove_linear_duplicates=remove_linear_duplicates, + end_to_end=end_to_end, + normalisation_mode=normalisation_mode, + normalisation_binsize=normalisation_binsize, + peakcaller=peakcaller, + use_control=use_control, + extend_fragments=extend_fragments, + igg_scale_factor=igg_scale_factor, + seacr_peak_threshold=seacr_peak_threshold, + seacr_norm=seacr_norm, + seacr_stringent=seacr_stringent, + macs2_qvalue=macs2_qvalue, + macs2_pvalue=macs2_pvalue, + macs_gsize=macs_gsize, + macs2_narrow_peak=macs2_narrow_peak, + macs2_broad_cutoff=macs2_broad_cutoff, + consensus_peak_mode=consensus_peak_mode, + replicate_threshold=replicate_threshold, + igv_show_gene_names=igv_show_gene_names, + dt_qc_bam_binsize=dt_qc_bam_binsize, + dt_qc_corr_method=dt_qc_corr_method, + dt_heatmap_gene_beforelen=dt_heatmap_gene_beforelen, + dt_heatmap_gene_bodylen=dt_heatmap_gene_bodylen, + dt_heatmap_gene_afterlen=dt_heatmap_gene_afterlen, + dt_heatmap_peak_beforelen=dt_heatmap_peak_beforelen, + dt_heatmap_peak_afterlen=dt_heatmap_peak_afterlen, + dt_calc_all_matrix=dt_calc_all_matrix, + min_frip_overlap=min_frip_overlap, + min_peak_overlap=min_peak_overlap, + igv_sort_by_groups=igv_sort_by_groups, + singularity_pull_docker_container=singularity_pull_docker_container, + multiqc_methods_description=multiqc_methods_description, + ) + + +LaunchPlan( + nf_nf_core_cutandrun, + "Test Data", + { + "input": [ + SampleSheet( + group="h3k27me3", + replicate=1, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k27me3_rep1_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k27me3_rep1_r2.fastq.gz"), + control="igg", + ), + SampleSheet( + group="h3k27me3", + replicate=2, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k27me3_rep2_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k27me3_rep2_r2.fastq.gz"), + control="igg", + ), + SampleSheet( + group="h3k4me3", + replicate=1, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k4me3_rep1_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k4me3_rep1_r2.fastq.gz"), + control="igg", + ), + SampleSheet( + group="h3k4me3", + replicate=2, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k4me3_rep2_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/h3k4me3_rep2_r2.fastq.gz"), + control="igg", + ), + SampleSheet( + group="igg", + replicate=1, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep1_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep1_r2.fastq.gz"), + control="", + ), + SampleSheet( + group="igg", + replicate=2, + fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep2_r1.fastq.gz"), + fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep2_r2.fastq.gz"), + control="", + ), + ], + "blacklist": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-blacklist.bed"), + "spikein_bowtie2": LatchFile("s3://latch-public/test-data/35729/cutandrun/e_coli_U00096_3.tar.gz"), + "spikein_fasta": LatchFile("s3://latch-public/test-data/35729/cutandrun/e_coli_U00096_3.fa"), + "fasta": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20.fa"), + "bowtie2": LatchDir("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-bowtie2/"), + "gtf": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-genes.gtf"), + "gene_bed": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-genes.bed"), + "run_name": "test-run", + "outdir": LatchOutputDir("latch:///cutandrun_test/"), + "peakcaller": "seacr", + "mito_name": "chrM", + "remove_linear_duplicates": True, + "remove_mitochondrial_reads": True, + }, +) diff --git a/wf/entrypoint.py b/wf/entrypoint.py index 98eae829..f439fdd2 100644 --- a/wf/entrypoint.py +++ b/wf/entrypoint.py @@ -1,17 +1,21 @@ import os import shutil import subprocess +import sys import typing +from dataclasses import dataclass +from enum import Enum from pathlib import Path import requests import typing_extensions from flytekit.core.annotation import FlyteAnnotation +from latch.executions import report_nextflow_used_storage from latch.ldata.path import LPath from latch.resources.tasks import custom_task, nextflow_runtime_task from latch.resources.workflow import workflow from latch.types import metadata -from latch.types.directory import LatchDir +from latch.types.directory import LatchDir, LatchOutputDir from latch.types.file import LatchFile from latch_cli.nextflow.utils import _get_execution_name from latch_cli.nextflow.workflow import get_flag @@ -20,6 +24,7 @@ meta = Path("latch_metadata") / "__init__.py" import_module_by_path(meta) +import latch_metadata @custom_task(cpu=0.25, memory=0.5, storage_gib=1) @@ -34,9 +39,7 @@ def initialize() -> str: resp = requests.post( "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage", headers=headers, - json={ - "storage_gib": 100, - }, + json={"storage_expiration_hours": 0, "version": 2}, ) resp.raise_for_status() print("Done.") @@ -44,50 +47,70 @@ def initialize() -> str: return resp.json()["name"] +@dataclass +class SampleSheet: + group: str + replicate: int + fastq_1: LatchFile + fastq_2: LatchFile + control: str + + +class Reference(Enum): + hg19 = "GRCh37 (Homo Sapiens hg19)" + hg38 = "GRCh38 (Homo Sapiens hg38)" + + +input_construct_samplesheet = metadata._nextflow_metadata.parameters["input"].samplesheet_constructor + + @nextflow_runtime_task(cpu=4, memory=8, storage_gib=100) def nextflow_runtime( pvc_name: str, - input: LatchFile, - outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], + input: typing.List[SampleSheet], + outdir: LatchOutputDir, multiqc_title: typing.Optional[str], - save_reference: typing.Optional[bool], - save_merged_fastq: typing.Optional[bool], - save_trimmed: typing.Optional[bool], - save_spikein_aligned: typing.Optional[bool], - save_unaligned: typing.Optional[bool], - save_align_intermed: typing.Optional[bool], + save_reference: bool, + save_merged_fastq: bool, + save_trimmed: bool, + save_spikein_aligned: bool, + save_unaligned: bool, + save_align_intermed: bool, email: typing.Optional[str], - dump_scale_factors: typing.Optional[bool], + dump_scale_factors: bool, + genome_source: str, genome: typing.Optional[str], - bowtie2: typing.Optional[str], - gtf: typing.Optional[str], - gene_bed: typing.Optional[str], - blacklist: typing.Optional[str], - spikein_bowtie2: typing.Optional[str], - spikein_fasta: typing.Optional[str], + latch_genome: typing.Optional[Reference], + bowtie2: typing.Optional[LatchDir], + gtf: typing.Optional[LatchFile], + gene_bed: typing.Optional[LatchFile], + blacklist: typing.Optional[LatchFile], + spikein_bowtie2: typing.Optional[LatchFile], + spikein_fasta: typing.Optional[LatchFile], fasta: typing.Optional[LatchFile], - only_input: typing.Optional[bool], - only_genome: typing.Optional[bool], - only_preqc: typing.Optional[bool], - only_alignment: typing.Optional[bool], - only_filtering: typing.Optional[bool], - only_peak_calling: typing.Optional[bool], - skip_fastqc: typing.Optional[bool], - skip_trimming: typing.Optional[bool], - skip_removeduplicates: typing.Optional[bool], - skip_reporting: typing.Optional[bool], - skip_preseq: typing.Optional[bool], - skip_igv: typing.Optional[bool], - skip_dt_qc: typing.Optional[bool], - skip_heatmaps: typing.Optional[bool], - skip_peak_qc: typing.Optional[bool], - skip_multiqc: typing.Optional[bool], - remove_mitochondrial_reads: typing.Optional[bool], + only_input: bool, + only_genome: bool, + only_preqc: bool, + only_alignment: bool, + only_filtering: bool, + only_peak_calling: bool, + skip_fastqc: bool, + skip_trimming: bool, + skip_removeduplicates: bool, + skip_reporting: bool, + skip_preseq: bool, + skip_igv: bool, + skip_dt_qc: bool, + skip_heatmaps: bool, + skip_peak_qc: bool, + skip_multiqc: bool, + remove_mitochondrial_reads: bool, + run_name: str, mito_name: typing.Optional[str], - dedup_target_reads: typing.Optional[bool], - remove_linear_duplicates: typing.Optional[bool], + dedup_target_reads: bool, + remove_linear_duplicates: bool, macs2_pvalue: typing.Optional[float], - singularity_pull_docker_container: typing.Optional[bool], + singularity_pull_docker_container: bool, multiqc_methods_description: typing.Optional[str], spikein_genome: typing.Optional[str], clip_r1: typing.Optional[int], @@ -96,23 +119,23 @@ def nextflow_runtime( three_prime_clip_r2: typing.Optional[int], trim_nextseq: typing.Optional[int], minimum_alignment_q_score: typing.Optional[int], - end_to_end: typing.Optional[bool], + end_to_end: bool, normalisation_mode: typing.Optional[str], normalisation_binsize: typing.Optional[int], peakcaller: typing.Optional[str], - use_control: typing.Optional[bool], - extend_fragments: typing.Optional[bool], + use_control: bool, + extend_fragments: bool, igg_scale_factor: typing.Optional[float], seacr_peak_threshold: typing.Optional[float], seacr_norm: typing.Optional[str], seacr_stringent: typing.Optional[str], macs2_qvalue: typing.Optional[float], macs_gsize: typing.Optional[float], - macs2_narrow_peak: typing.Optional[bool], + macs2_narrow_peak: bool, macs2_broad_cutoff: typing.Optional[float], consensus_peak_mode: typing.Optional[str], replicate_threshold: typing.Optional[float], - igv_show_gene_names: typing.Optional[bool], + igv_show_gene_names: bool, dt_qc_bam_binsize: typing.Optional[int], dt_qc_corr_method: typing.Optional[str], dt_heatmap_gene_beforelen: typing.Optional[int], @@ -120,133 +143,160 @@ def nextflow_runtime( dt_heatmap_gene_afterlen: typing.Optional[int], dt_heatmap_peak_beforelen: typing.Optional[int], dt_heatmap_peak_afterlen: typing.Optional[int], - dt_calc_all_matrix: typing.Optional[bool], + dt_calc_all_matrix: bool, min_frip_overlap: typing.Optional[float], min_peak_overlap: typing.Optional[float], - igv_sort_by_groups: typing.Optional[bool], + igv_sort_by_groups: bool, ) -> None: - try: - shared_dir = Path("/nf-workdir") + shared_dir = Path("/nf-workdir") + + input_samplesheet = input_construct_samplesheet(input) + + ignore_list = [ + "latch", + ".latch", + ".git", + "nextflow", + ".nextflow", + "work", + "results", + "miniconda", + "anaconda3", + "mambaforge", + ] + + shutil.copytree( + Path("/root"), + shared_dir, + ignore=lambda src, names: ignore_list, + ignore_dangling_symlinks=True, + dirs_exist_ok=True, + ) + + profile_list = ["docker"] + if False: + profile_list.extend([p.value for p in execution_profiles]) - ignore_list = [ - "latch", - ".latch", - "nextflow", - ".nextflow", - "work", - "results", - "miniconda", - "anaconda3", - "mambaforge", - ] + if len(profile_list) == 0: + profile_list.append("standard") - shutil.copytree( - Path("/root"), - shared_dir, - ignore=lambda src, names: ignore_list, - ignore_dangling_symlinks=True, - dirs_exist_ok=True, + profiles = ",".join(profile_list) + if genome_source == "latch_genome_source": + fasta = os.path.join( + "s3://latch-public/test-data/35729/cutandrun/", + latch_genome.name, + latch_genome.name + ".fa", ) + gtf = os.path.join( + "s3://latch-public/test-data/35729/cutandrun/", + latch_genome.name, + latch_genome.name + ".refGene.gtf", + ) + bowtie2 = os.path.join("s3://latch-public/test-data/35729/cutandrun/", latch_genome.name, "bowtie2") - cmd = [ - "/root/nextflow", - "run", - str(shared_dir / "main.nf"), - "-work-dir", - str(shared_dir), - "-profile", - "docker", - "-c", - "latch.config", - *get_flag("input", input), - *get_flag("outdir", outdir), - *get_flag("multiqc_title", multiqc_title), - *get_flag("save_reference", save_reference), - *get_flag("save_merged_fastq", save_merged_fastq), - *get_flag("save_trimmed", save_trimmed), - *get_flag("save_spikein_aligned", save_spikein_aligned), - *get_flag("save_unaligned", save_unaligned), - *get_flag("save_align_intermed", save_align_intermed), - *get_flag("email", email), - *get_flag("dump_scale_factors", dump_scale_factors), - *get_flag("genome", genome), - *get_flag("bowtie2", bowtie2), - *get_flag("gtf", gtf), - *get_flag("gene_bed", gene_bed), - *get_flag("blacklist", blacklist), - *get_flag("spikein_genome", spikein_genome), - *get_flag("spikein_bowtie2", spikein_bowtie2), - *get_flag("spikein_fasta", spikein_fasta), - *get_flag("fasta", fasta), - *get_flag("only_input", only_input), - *get_flag("only_genome", only_genome), - *get_flag("only_preqc", only_preqc), - *get_flag("only_alignment", only_alignment), - *get_flag("only_filtering", only_filtering), - *get_flag("only_peak_calling", only_peak_calling), - *get_flag("skip_fastqc", skip_fastqc), - *get_flag("skip_trimming", skip_trimming), - *get_flag("skip_removeduplicates", skip_removeduplicates), - *get_flag("skip_reporting", skip_reporting), - *get_flag("skip_preseq", skip_preseq), - *get_flag("skip_igv", skip_igv), - *get_flag("skip_dt_qc", skip_dt_qc), - *get_flag("skip_heatmaps", skip_heatmaps), - *get_flag("skip_peak_qc", skip_peak_qc), - *get_flag("skip_multiqc", skip_multiqc), - *get_flag("clip_r1", clip_r1), - *get_flag("clip_r2", clip_r2), - *get_flag("three_prime_clip_r1", three_prime_clip_r1), - *get_flag("three_prime_clip_r2", three_prime_clip_r2), - *get_flag("trim_nextseq", trim_nextseq), - *get_flag("minimum_alignment_q_score", minimum_alignment_q_score), - *get_flag("remove_mitochondrial_reads", remove_mitochondrial_reads), - *get_flag("mito_name", mito_name), - *get_flag("dedup_target_reads", dedup_target_reads), - *get_flag("remove_linear_duplicates", remove_linear_duplicates), - *get_flag("end_to_end", end_to_end), - *get_flag("normalisation_mode", normalisation_mode), - *get_flag("normalisation_binsize", normalisation_binsize), - *get_flag("peakcaller", peakcaller), - *get_flag("use_control", use_control), - *get_flag("extend_fragments", extend_fragments), - *get_flag("igg_scale_factor", igg_scale_factor), - *get_flag("seacr_peak_threshold", seacr_peak_threshold), - *get_flag("seacr_norm", seacr_norm), - *get_flag("seacr_stringent", seacr_stringent), - *get_flag("macs2_qvalue", macs2_qvalue), - *get_flag("macs2_pvalue", macs2_pvalue), - *get_flag("macs_gsize", macs_gsize), - *get_flag("macs2_narrow_peak", macs2_narrow_peak), - *get_flag("macs2_broad_cutoff", macs2_broad_cutoff), - *get_flag("consensus_peak_mode", consensus_peak_mode), - *get_flag("replicate_threshold", replicate_threshold), - *get_flag("igv_show_gene_names", igv_show_gene_names), - *get_flag("dt_qc_bam_binsize", dt_qc_bam_binsize), - *get_flag("dt_qc_corr_method", dt_qc_corr_method), - *get_flag("dt_heatmap_gene_beforelen", dt_heatmap_gene_beforelen), - *get_flag("dt_heatmap_gene_bodylen", dt_heatmap_gene_bodylen), - *get_flag("dt_heatmap_gene_afterlen", dt_heatmap_gene_afterlen), - *get_flag("dt_heatmap_peak_beforelen", dt_heatmap_peak_beforelen), - *get_flag("dt_heatmap_peak_afterlen", dt_heatmap_peak_afterlen), - *get_flag("dt_calc_all_matrix", dt_calc_all_matrix), - *get_flag("min_frip_overlap", min_frip_overlap), - *get_flag("min_peak_overlap", min_peak_overlap), - *get_flag("igv_sort_by_groups", igv_sort_by_groups), - *get_flag("singularity_pull_docker_container", singularity_pull_docker_container), - *get_flag("multiqc_methods_description", multiqc_methods_description), - ] + cmd = [ + "/root/nextflow", + "run", + str(shared_dir / "main.nf"), + "-work-dir", + str(shared_dir), + "-profile", + profiles, + "-c", + "latch.config", + "-resume", + *get_flag("input", input_samplesheet), + *get_flag("outdir", LatchOutputDir(outdir.remote_path + "/" + run_name)), + *get_flag("multiqc_title", multiqc_title), + *get_flag("save_reference", save_reference), + *get_flag("save_merged_fastq", save_merged_fastq), + *get_flag("save_trimmed", save_trimmed), + *get_flag("save_spikein_aligned", save_spikein_aligned), + *get_flag("save_unaligned", save_unaligned), + *get_flag("save_align_intermed", save_align_intermed), + *get_flag("email", email), + *get_flag("dump_scale_factors", dump_scale_factors), + *get_flag("genome", genome), + *get_flag("bowtie2", bowtie2), + *get_flag("gtf", gtf), + *get_flag("gene_bed", gene_bed), + *get_flag("blacklist", blacklist), + *get_flag("spikein_genome", spikein_genome), + *get_flag("spikein_bowtie2", spikein_bowtie2), + *get_flag("spikein_fasta", spikein_fasta), + *get_flag("fasta", fasta), + *get_flag("only_input", only_input), + *get_flag("only_genome", only_genome), + *get_flag("only_preqc", only_preqc), + *get_flag("only_alignment", only_alignment), + *get_flag("only_filtering", only_filtering), + *get_flag("only_peak_calling", only_peak_calling), + *get_flag("skip_fastqc", skip_fastqc), + *get_flag("skip_trimming", skip_trimming), + *get_flag("skip_removeduplicates", skip_removeduplicates), + *get_flag("skip_reporting", skip_reporting), + *get_flag("skip_preseq", skip_preseq), + *get_flag("skip_igv", skip_igv), + *get_flag("skip_dt_qc", skip_dt_qc), + *get_flag("skip_heatmaps", skip_heatmaps), + *get_flag("skip_peak_qc", skip_peak_qc), + *get_flag("skip_multiqc", skip_multiqc), + *get_flag("clip_r1", clip_r1), + *get_flag("clip_r2", clip_r2), + *get_flag("three_prime_clip_r1", three_prime_clip_r1), + *get_flag("three_prime_clip_r2", three_prime_clip_r2), + *get_flag("trim_nextseq", trim_nextseq), + *get_flag("minimum_alignment_q_score", minimum_alignment_q_score), + *get_flag("remove_mitochondrial_reads", remove_mitochondrial_reads), + *get_flag("mito_name", mito_name), + *get_flag("dedup_target_reads", dedup_target_reads), + *get_flag("remove_linear_duplicates", remove_linear_duplicates), + *get_flag("end_to_end", end_to_end), + *get_flag("normalisation_mode", normalisation_mode), + *get_flag("normalisation_binsize", normalisation_binsize), + *get_flag("peakcaller", peakcaller), + *get_flag("use_control", use_control), + *get_flag("extend_fragments", extend_fragments), + *get_flag("igg_scale_factor", igg_scale_factor), + *get_flag("seacr_peak_threshold", seacr_peak_threshold), + *get_flag("seacr_norm", seacr_norm), + *get_flag("seacr_stringent", seacr_stringent), + *get_flag("macs2_qvalue", macs2_qvalue), + *get_flag("macs2_pvalue", macs2_pvalue), + *get_flag("macs_gsize", macs_gsize), + *get_flag("macs2_narrow_peak", macs2_narrow_peak), + *get_flag("macs2_broad_cutoff", macs2_broad_cutoff), + *get_flag("consensus_peak_mode", consensus_peak_mode), + *get_flag("replicate_threshold", replicate_threshold), + *get_flag("igv_show_gene_names", igv_show_gene_names), + *get_flag("dt_qc_bam_binsize", dt_qc_bam_binsize), + *get_flag("dt_qc_corr_method", dt_qc_corr_method), + *get_flag("dt_heatmap_gene_beforelen", dt_heatmap_gene_beforelen), + *get_flag("dt_heatmap_gene_bodylen", dt_heatmap_gene_bodylen), + *get_flag("dt_heatmap_gene_afterlen", dt_heatmap_gene_afterlen), + *get_flag("dt_heatmap_peak_beforelen", dt_heatmap_peak_beforelen), + *get_flag("dt_heatmap_peak_afterlen", dt_heatmap_peak_afterlen), + *get_flag("dt_calc_all_matrix", dt_calc_all_matrix), + *get_flag("min_frip_overlap", min_frip_overlap), + *get_flag("min_peak_overlap", min_peak_overlap), + *get_flag("igv_sort_by_groups", igv_sort_by_groups), + *get_flag("singularity_pull_docker_container", singularity_pull_docker_container), + *get_flag("multiqc_methods_description", multiqc_methods_description), + ] - print("Launching Nextflow Runtime") - print(" ".join(cmd)) - print(flush=True) + print("Launching Nextflow Runtime") + print(" ".join(cmd)) + print(flush=True) + failed = False + try: env = { **os.environ, + "NXF_ANSI_LOG": "false", "NXF_HOME": "/root/.nextflow", - "NXF_OPTS": "-Xms2048M -Xmx8G -XX:ActiveProcessorCount=4", - "K8S_STORAGE_CLAIM_NAME": pvc_name, + "NXF_OPTS": "-Xms1536M -Xmx6144M -XX:ActiveProcessorCount=4", "NXF_DISABLE_CHECK_LATEST": "true", + "NXF_ENABLE_VIRTUAL_THREADS": "false", } subprocess.run( cmd, @@ -254,6 +304,8 @@ def nextflow_runtime( check=True, cwd=str(shared_dir), ) + except subprocess.CalledProcessError: + failed = True finally: print() @@ -267,171 +319,26 @@ def nextflow_runtime( print(f"Uploading .nextflow.log to {remote.path}") remote.upload_from(nextflow_log) + print("Computing size of workdir... ", end="") + try: + result = subprocess.run( + ["du", "-sb", str(shared_dir)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=5 * 60, + ) -@workflow(metadata._nextflow_metadata) -def nf_nf_core_cutandrun( - input: LatchFile, - outdir: typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], - multiqc_title: typing.Optional[str], - save_reference: typing.Optional[bool], - save_merged_fastq: typing.Optional[bool], - save_trimmed: typing.Optional[bool], - save_spikein_aligned: typing.Optional[bool], - save_unaligned: typing.Optional[bool], - save_align_intermed: typing.Optional[bool], - email: typing.Optional[str], - dump_scale_factors: typing.Optional[bool], - genome: typing.Optional[str], - bowtie2: typing.Optional[str], - gtf: typing.Optional[str], - gene_bed: typing.Optional[str], - blacklist: typing.Optional[str], - spikein_bowtie2: typing.Optional[str], - spikein_fasta: typing.Optional[str], - fasta: typing.Optional[LatchFile], - only_input: typing.Optional[bool], - only_genome: typing.Optional[bool], - only_preqc: typing.Optional[bool], - only_alignment: typing.Optional[bool], - only_filtering: typing.Optional[bool], - only_peak_calling: typing.Optional[bool], - skip_fastqc: typing.Optional[bool], - skip_trimming: typing.Optional[bool], - skip_removeduplicates: typing.Optional[bool], - skip_reporting: typing.Optional[bool], - skip_preseq: typing.Optional[bool], - skip_igv: typing.Optional[bool], - skip_dt_qc: typing.Optional[bool], - skip_heatmaps: typing.Optional[bool], - skip_peak_qc: typing.Optional[bool], - skip_multiqc: typing.Optional[bool], - remove_mitochondrial_reads: typing.Optional[bool], - mito_name: typing.Optional[str], - dedup_target_reads: typing.Optional[bool], - remove_linear_duplicates: typing.Optional[bool], - macs2_pvalue: typing.Optional[float], - singularity_pull_docker_container: typing.Optional[bool], - multiqc_methods_description: typing.Optional[str], - spikein_genome: typing.Optional[str] = "K12-MG1655", - clip_r1: typing.Optional[int] = 0, - clip_r2: typing.Optional[int] = 0, - three_prime_clip_r1: typing.Optional[int] = 0, - three_prime_clip_r2: typing.Optional[int] = 0, - trim_nextseq: typing.Optional[int] = 0, - minimum_alignment_q_score: typing.Optional[int] = 20, - end_to_end: typing.Optional[bool] = True, - normalisation_mode: typing.Optional[str] = "Spikein", - normalisation_binsize: typing.Optional[int] = 50, - peakcaller: typing.Optional[str] = "seacr", - use_control: typing.Optional[bool] = True, - extend_fragments: typing.Optional[bool] = True, - igg_scale_factor: typing.Optional[float] = 0.5, - seacr_peak_threshold: typing.Optional[float] = 0.05, - seacr_norm: typing.Optional[str] = "non", - seacr_stringent: typing.Optional[str] = "stringent", - macs2_qvalue: typing.Optional[float] = 0.01, - macs_gsize: typing.Optional[float] = 2700000000.0, - macs2_narrow_peak: typing.Optional[bool] = True, - macs2_broad_cutoff: typing.Optional[float] = 0.1, - consensus_peak_mode: typing.Optional[str] = "group", - replicate_threshold: typing.Optional[float] = 1.0, - igv_show_gene_names: typing.Optional[bool] = True, - dt_qc_bam_binsize: typing.Optional[int] = 500, - dt_qc_corr_method: typing.Optional[str] = "pearson", - dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, - dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, - dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, - dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, - dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, - dt_calc_all_matrix: typing.Optional[bool] = True, - min_frip_overlap: typing.Optional[float] = 0.2, - min_peak_overlap: typing.Optional[float] = 0.2, - igv_sort_by_groups: typing.Optional[bool] = True, -) -> None: - """ - nf-core/cutandrun - - Sample Description - """ + size = int(result.stdout.split()[0]) + report_nextflow_used_storage(size) + print(f"Done. Workdir size: {size / 1024 / 1024 / 1024: .2f} GiB") + except subprocess.TimeoutExpired: + print("Failed to compute storage size: Operation timed out after 5 minutes.") + except subprocess.CalledProcessError as e: + print(f"Failed to compute storage size: {e.stderr}") + except Exception as e: + print(f"Failed to compute storage size: {e}") - pvc_name: str = initialize() - nextflow_runtime( - pvc_name=pvc_name, - input=input, - outdir=outdir, - multiqc_title=multiqc_title, - save_reference=save_reference, - save_merged_fastq=save_merged_fastq, - save_trimmed=save_trimmed, - save_spikein_aligned=save_spikein_aligned, - save_unaligned=save_unaligned, - save_align_intermed=save_align_intermed, - email=email, - dump_scale_factors=dump_scale_factors, - genome=genome, - bowtie2=bowtie2, - gtf=gtf, - gene_bed=gene_bed, - blacklist=blacklist, - spikein_genome=spikein_genome, - spikein_bowtie2=spikein_bowtie2, - spikein_fasta=spikein_fasta, - fasta=fasta, - only_input=only_input, - only_genome=only_genome, - only_preqc=only_preqc, - only_alignment=only_alignment, - only_filtering=only_filtering, - only_peak_calling=only_peak_calling, - skip_fastqc=skip_fastqc, - skip_trimming=skip_trimming, - skip_removeduplicates=skip_removeduplicates, - skip_reporting=skip_reporting, - skip_preseq=skip_preseq, - skip_igv=skip_igv, - skip_dt_qc=skip_dt_qc, - skip_heatmaps=skip_heatmaps, - skip_peak_qc=skip_peak_qc, - skip_multiqc=skip_multiqc, - clip_r1=clip_r1, - clip_r2=clip_r2, - three_prime_clip_r1=three_prime_clip_r1, - three_prime_clip_r2=three_prime_clip_r2, - trim_nextseq=trim_nextseq, - minimum_alignment_q_score=minimum_alignment_q_score, - remove_mitochondrial_reads=remove_mitochondrial_reads, - mito_name=mito_name, - dedup_target_reads=dedup_target_reads, - remove_linear_duplicates=remove_linear_duplicates, - end_to_end=end_to_end, - normalisation_mode=normalisation_mode, - normalisation_binsize=normalisation_binsize, - peakcaller=peakcaller, - use_control=use_control, - extend_fragments=extend_fragments, - igg_scale_factor=igg_scale_factor, - seacr_peak_threshold=seacr_peak_threshold, - seacr_norm=seacr_norm, - seacr_stringent=seacr_stringent, - macs2_qvalue=macs2_qvalue, - macs2_pvalue=macs2_pvalue, - macs_gsize=macs_gsize, - macs2_narrow_peak=macs2_narrow_peak, - macs2_broad_cutoff=macs2_broad_cutoff, - consensus_peak_mode=consensus_peak_mode, - replicate_threshold=replicate_threshold, - igv_show_gene_names=igv_show_gene_names, - dt_qc_bam_binsize=dt_qc_bam_binsize, - dt_qc_corr_method=dt_qc_corr_method, - dt_heatmap_gene_beforelen=dt_heatmap_gene_beforelen, - dt_heatmap_gene_bodylen=dt_heatmap_gene_bodylen, - dt_heatmap_gene_afterlen=dt_heatmap_gene_afterlen, - dt_heatmap_peak_beforelen=dt_heatmap_peak_beforelen, - dt_heatmap_peak_afterlen=dt_heatmap_peak_afterlen, - dt_calc_all_matrix=dt_calc_all_matrix, - min_frip_overlap=min_frip_overlap, - min_peak_overlap=min_peak_overlap, - igv_sort_by_groups=igv_sort_by_groups, - singularity_pull_docker_container=singularity_pull_docker_container, - multiqc_methods_description=multiqc_methods_description, - ) + if failed: + sys.exit(1) From c3805f5d6a92b7c20510c4d2414527cfe0ed56e1 Mon Sep 17 00:00:00 2001 From: Tahir D'Mello Date: Sat, 14 Sep 2024 12:19:11 -0700 Subject: [PATCH 4/5] ui cleanup --- .gitignore | 2 + latch_metadata/__init__.py | 23 +++- latch_metadata/parameters.py | 24 ++-- version | 2 +- wf/__init__.py | 209 ++++++++++++++++++++++++++--------- wf/entrypoint.py | 117 ++++++++++---------- 6 files changed, 241 insertions(+), 136 deletions(-) diff --git a/.gitignore b/.gitignore index 441060fb..4364d586 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ R_sessionInfo.log .vscode/ profile.svg + +.latch/.logs diff --git a/latch_metadata/__init__.py b/latch_metadata/__init__.py index c6c0f146..f39b7ffa 100644 --- a/latch_metadata/__init__.py +++ b/latch_metadata/__init__.py @@ -1,13 +1,9 @@ -from latch.ldata.path import LPath -from latch.types import metadata -from latch.types.directory import LatchDir, LatchOutputDir -from latch.types.file import LatchFile +from latch.types.directory import LatchDir from latch.types.metadata import ( Fork, ForkBranch, LatchAuthor, NextflowMetadata, - NextflowParameter, NextflowRuntimeResources, Params, Section, @@ -18,7 +14,14 @@ from .parameters import generated_parameters flow = [ - Section("Input/Output", Params("input", "outdir", "run_name"), Spoiler("Optional", Params("use_control"))), + Section( + "Input", + Params("input"), + Spoiler( + "Optional", + Params("use_control"), + ), + ), Section( "Reference Genome Options", Fork( @@ -48,6 +51,12 @@ "save_spikein_aligned", ), ), + Section( + "Output Directory", + Params("run_name"), + Text("Parent directory for outputs"), + Params("outdir"), + ), Spoiler( "Optional Parameters", Spoiler( @@ -159,11 +168,13 @@ ), ] + NextflowMetadata( display_name="nf-core/cutandrun", author=LatchAuthor( name="nf-core", ), + repository="https://github.com/latchbio-nfcore/cutandrun", parameters=generated_parameters, runtime_resources=NextflowRuntimeResources( cpus=4, diff --git a/latch_metadata/parameters.py b/latch_metadata/parameters.py index c2a58281..180076f7 100644 --- a/latch_metadata/parameters.py +++ b/latch_metadata/parameters.py @@ -1,17 +1,10 @@ -import csv import typing from dataclasses import dataclass from enum import Enum -import typing_extensions -from flytekit.core.annotation import FlyteAnnotation from latch.types.directory import LatchDir, LatchOutputDir from latch.types.file import LatchFile -from latch.types.metadata import Multiselect, NextflowParameter - -# Import these into your `__init__.py` file: -# -# from .parameters import generated_parameters +from latch.types.metadata import LatchRule, Multiselect, NextflowParameter @dataclass @@ -24,8 +17,8 @@ class SampleSheet: class Reference(Enum): - hg19 = "GRCh37 (Homo Sapiens hg19)" hg38 = "GRCh38 (Homo Sapiens hg38)" + hg19 = "GRCh37 (Homo Sapiens hg19)" generated_parameters = { @@ -39,7 +32,7 @@ class Reference(Enum): default=None, ), "outdir": NextflowParameter( - type=LatchOutputDir, # typing_extensions.Annotated[LatchDir, FlyteAnnotation({"output": True})], + type=LatchOutputDir, default=None, display_name="Outdir", description="The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.", @@ -345,10 +338,15 @@ class Reference(Enum): ), "run_name": NextflowParameter( type=str, - default=None, - section_title=None, display_name="Run Name", - description="Run Name", + description="Name of run", + batch_table_column=True, + rules=[ + LatchRule( + regex=r"^[a-zA-Z0-9_-]+$", + message="Run name must contain only letters, digits, underscores, and dashes. No spaces are allowed.", + ) + ], ), "mito_name": NextflowParameter( type=typing.Optional[str], diff --git a/version b/version index 6e8bf73a..be94e6f5 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.0 +3.2.2 diff --git a/wf/__init__.py b/wf/__init__.py index 2d1e9244..b8e7c822 100644 --- a/wf/__init__.py +++ b/wf/__init__.py @@ -1,11 +1,6 @@ -import typing +from typing import List, Optional -import typing_extensions -from flytekit.core.annotation import FlyteAnnotation -from latch.executions import report_nextflow_used_storage -from latch.ldata.path import LPath from latch.resources.launch_plan import LaunchPlan -from latch.resources.tasks import custom_task, nextflow_runtime_task from latch.resources.workflow import workflow from latch.types import metadata from latch.types.directory import LatchDir, LatchOutputDir @@ -16,27 +11,26 @@ @workflow(metadata._nextflow_metadata) def nf_nf_core_cutandrun( - input: typing.List[SampleSheet], - outdir: LatchOutputDir, - multiqc_title: typing.Optional[str], + input: List[SampleSheet], + multiqc_title: Optional[str], save_reference: bool, save_merged_fastq: bool, save_trimmed: bool, save_spikein_aligned: bool, save_unaligned: bool, save_align_intermed: bool, - email: typing.Optional[str], + email: Optional[str], dump_scale_factors: bool, genome_source: str, - genome: typing.Optional[str], - latch_genome: typing.Optional[Reference], - bowtie2: typing.Optional[LatchDir], - gtf: typing.Optional[LatchFile], - gene_bed: typing.Optional[LatchFile], - blacklist: typing.Optional[LatchFile], - spikein_bowtie2: typing.Optional[LatchFile], - spikein_fasta: typing.Optional[LatchFile], - fasta: typing.Optional[LatchFile], + genome: Optional[str], + latch_genome: Optional[Reference], + bowtie2: Optional[LatchDir], + gtf: Optional[LatchFile], + gene_bed: Optional[LatchFile], + blacklist: Optional[LatchFile], + spikein_bowtie2: Optional[LatchFile], + spikein_fasta: Optional[LatchFile], + fasta: Optional[LatchFile], only_input: bool, only_genome: bool, only_preqc: bool, @@ -55,55 +49,160 @@ def nf_nf_core_cutandrun( skip_multiqc: bool, remove_mitochondrial_reads: bool, run_name: str, - mito_name: typing.Optional[str], + mito_name: Optional[str], dedup_target_reads: bool, remove_linear_duplicates: bool, - macs2_pvalue: typing.Optional[float], + macs2_pvalue: Optional[float], singularity_pull_docker_container: bool, - multiqc_methods_description: typing.Optional[str], - spikein_genome: typing.Optional[str] = "K12-MG1655", - clip_r1: typing.Optional[int] = 0, - clip_r2: typing.Optional[int] = 0, - three_prime_clip_r1: typing.Optional[int] = 0, - three_prime_clip_r2: typing.Optional[int] = 0, - trim_nextseq: typing.Optional[int] = 0, - minimum_alignment_q_score: typing.Optional[int] = 20, + multiqc_methods_description: Optional[str], + spikein_genome: Optional[str] = "K12-MG1655", + clip_r1: Optional[int] = 0, + clip_r2: Optional[int] = 0, + three_prime_clip_r1: Optional[int] = 0, + three_prime_clip_r2: Optional[int] = 0, + trim_nextseq: Optional[int] = 0, + minimum_alignment_q_score: Optional[int] = 20, end_to_end: bool = True, - normalisation_mode: typing.Optional[str] = "Spikein", - normalisation_binsize: typing.Optional[int] = 50, - peakcaller: typing.Optional[str] = "seacr", + normalisation_mode: Optional[str] = "Spikein", + normalisation_binsize: Optional[int] = 50, + peakcaller: Optional[str] = "seacr", use_control: bool = True, extend_fragments: bool = True, - igg_scale_factor: typing.Optional[float] = 0.5, - seacr_peak_threshold: typing.Optional[float] = 0.05, - seacr_norm: typing.Optional[str] = "non", - seacr_stringent: typing.Optional[str] = "stringent", - macs2_qvalue: typing.Optional[float] = 0.01, - macs_gsize: typing.Optional[float] = 2700000000.0, + igg_scale_factor: Optional[float] = 0.5, + seacr_peak_threshold: Optional[float] = 0.05, + seacr_norm: Optional[str] = "non", + seacr_stringent: Optional[str] = "stringent", + macs2_qvalue: Optional[float] = 0.01, + macs_gsize: Optional[float] = 2700000000.0, macs2_narrow_peak: bool = True, - macs2_broad_cutoff: typing.Optional[float] = 0.1, - consensus_peak_mode: typing.Optional[str] = "group", - replicate_threshold: typing.Optional[float] = 1.0, + macs2_broad_cutoff: Optional[float] = 0.1, + consensus_peak_mode: Optional[str] = "group", + replicate_threshold: Optional[float] = 1.0, igv_show_gene_names: bool = True, - dt_qc_bam_binsize: typing.Optional[int] = 500, - dt_qc_corr_method: typing.Optional[str] = "pearson", - dt_heatmap_gene_beforelen: typing.Optional[int] = 3000, - dt_heatmap_gene_bodylen: typing.Optional[int] = 5000, - dt_heatmap_gene_afterlen: typing.Optional[int] = 3000, - dt_heatmap_peak_beforelen: typing.Optional[int] = 3000, - dt_heatmap_peak_afterlen: typing.Optional[int] = 3000, + dt_qc_bam_binsize: Optional[int] = 500, + dt_qc_corr_method: Optional[str] = "pearson", + dt_heatmap_gene_beforelen: Optional[int] = 3000, + dt_heatmap_gene_bodylen: Optional[int] = 5000, + dt_heatmap_gene_afterlen: Optional[int] = 3000, + dt_heatmap_peak_beforelen: Optional[int] = 3000, + dt_heatmap_peak_afterlen: Optional[int] = 3000, dt_calc_all_matrix: bool = True, - min_frip_overlap: typing.Optional[float] = 0.2, - min_peak_overlap: typing.Optional[float] = 0.2, + min_frip_overlap: Optional[float] = 0.2, + min_peak_overlap: Optional[float] = 0.2, igv_sort_by_groups: bool = True, + outdir: LatchOutputDir = LatchOutputDir("latch:///CutandRun"), ) -> None: """ - nf-core/cutandrun + nf-core/cutandrun is a best-practice bioinformatic analysis pipeline for CUT&RUN, CUT&Tag, and TIPseq experimental protocols that were developed to study protein-DNA interactions and epigenomic profiling. + + +

+ Latch Verified +

+ +

+ + Latch Verified + +

+ +

+ + [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5653535-1073c8)](https://doi.org/10.5281/zenodo.5653535) + + **nf-core/cutandrun** is a best-practice bioinformatic analysis pipeline for CUT&RUN, CUT&Tag, and TIPseq experimental protocols that were developed to study protein-DNA interactions and epigenomic profiling. + + [CUT&RUN](https://elifesciences.org/articles/46314) + + > Meers, M. P., Bryson, T. D., Henikoff, J. G., & Henikoff, S. (2019). Improved CUT&RUN chromatin profiling tools. _eLife_, _8_. https://doi.org/10.7554/eLife.46314 + + [CUT&Tag](https://www.nature.com/articles/s41467-019-09982-5) + + > Kaya-Okur, H. S., Wu, S. J., Codomo, C. A., Pledger, E. S., Bryson, T. D., Henikoff, J. G., Ahmad, K., & Henikoff, S. (2019). CUT&Tag for efficient epigenomic profiling of small samples and single cells. _Nature Communications_, _10_(1), 1930. https://doi.org/10.1038/s41467-019-09982-5] + + [TIPseq](https://rupress.org/jcb/article/220/12/e202103078/212821) + + > Bartlett, D. A., Dileep, V., Handa, T., Ohkawa, Y., Kimura, H., Henikoff, S., & Gilbert, D. M. (2021). High-throughput single-cell epigenomic profiling by targeted insertion of promoters (TIP-seq). Journal of Cell Biology, 220(12), e202103078. https://doi.org/10.1083/jcb.202103078 + + The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a portable, reproducible manner. It is capable of using containerisation and package management making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process, which makes it easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules). + + This workflow is hosted on Latch Workflows, using a native Nextflow integration, with a graphical interface for accessible analysis by scientists. There is also an integration with Latch Registry so that batched workflows can be launched from “graphical sample sheets” or tables associating raw sequencing files with metadata. + + ## Pipeline summary + + 1. Check input files + 2. Merge re-sequenced FastQ files ([`cat`](http://www.linfo.org/cat.html)) + 3. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) + 4. Adapter and quality trimming ([`Trim Galore!`](https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/)) + 5. Alignment to both target and spike-in genomes ([`Bowtie 2`](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml)) + 6. Filter on quality, sort and index alignments ([`samtools`](https://sourceforge.net/projects/samtools/files/samtools/)) + 7. Duplicate read marking ([`picard`](https://broadinstitute.github.io/picard/)) + 8. Create bedGraph files ([`bedtools`](https://github.com/arq5x/bedtools2/) + 9. Create bigWig coverage files ([`bedGraphToBigWig`](http://hgdownload.soe.ucsc.edu/admin/exe/)) + 10. Peak calling ([`SEACR`](https://github.com/FredHutch/SEACR), [`MACS2`](https://github.com/macs3-project/MACS)) + 11. Consensus peak merging and reporting ([`bedtools`](https://github.com/arq5x/bedtools2/)) + 12. Library complexity ([preseq]([Preseq | The Smith Lab](http://smithlabresearch.org/software/preseq))) + 13. Fragment-based quality control ([`deepTools`](https://github.com/deeptools/deepTools/)) + 14. Peak-based quality control ([`bedtools`](https://github.com/arq5x/bedtools2/), custom python) + 15. Heatmap peak analysis ([`deepTools`](https://github.com/deeptools/deepTools/)) + 16. Genome browser session ([`IGV`](https://software.broadinstitute.org/software/igv/)) + 17. Present all QC in web-based report ([`MultiQC`](http://multiqc.info/)) + + ## Usage + + First, prepare a samplesheet with your input data that looks as follows: + + `samplesheet.csv`: + + ``` + group,replicate,fastq_1,fastq_2,control + h3k27me3,1,h3k27me3_rep1_r1.fastq.gz,h3k27me3_rep1_r2.fastq.gz,igg_ctrl + h3k27me3,2,h3k27me3_rep2_r1.fastq.gz,h3k27me3_rep2_r2.fastq.gz,igg_ctrl + igg_ctrl,1,igg_rep1_r1.fastq.gz,igg_rep1_r2.fastq.gz, + igg_ctrl,2,igg_rep2_r1.fastq.gz,igg_rep2_r2.fastq.gz, + ``` + + Each row represents a pair of fastq files (paired end). + + ## Pipeline output + + To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/cutandrun/results) tab on the nf-core website pipeline page. + For more details about the output files and reports, please refer to the + [output documentation](https://nf-co.re/cutandrun/output). + + ## Credits + + nf-core/cutandrun was originally written by Chris Cheshire ([@chris-cheshire](https://github.com/chris-cheshire)) and Charlotte West ([@charlotte-west](https://github.com/charlotte-west)) from [Luscombe Lab](https://www.crick.ac.uk/research/labs/nicholas-luscombe) at [The Francis Crick Institute](https://www.crick.ac.uk/), London, UK. + + The pipeline structure and parts of the downstream analysis were adapted from the original CUT&Tag analysis [protocol](https://yezhengstat.github.io/CUTTag_tutorial/) from the [Henikoff Lab](https://research.fredhutch.org/henikoff/en.html). The removal of duplicates arising from linear amplification (also known as T7 duplicates) in the TIPseq protocol was implemented as described in the original [TIPseq paper](https://rupress.org/jcb/article/220/12/e202103078/212821). + + We thank Harshil Patel ([@drpatelh](https://github.com/drpatelh)) and everyone in the Luscombe Lab ([@luslab](https://github.com/luslab)) for their extensive assistance in the development of this pipeline. + + ![[The Francis Crick Institute](https://www.crick.ac.uk/)](docs/images/crick_logo.png) + + ## Contributions and Support + + If you would like to contribute to this pipeline, please see the [contributing guidelines](.github/CONTRIBUTING.md). + + For further information or help, don't hesitate to get in touch on the [Slack `#cutandrun` channel](https://nfcore.slack.com/channels/cutandrun) (you can join with [this invite](https://nf-co.re/join/slack)). + + ## Citations + + If you use nf-core/cutandrun for your analysis, please cite it using the following doi: [10.5281/zenodo.5653535](https://doi.org/10.5281/zenodo.5653535) + + An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file. + + You can cite the `nf-core` publication as follows: + + > **The nf-core framework for community-curated bioinformatics pipelines.** + > + > Philip Ewels, Alexander Peltzer, Sven Fillinger, Harshil Patel, Johannes Alneberg, Andreas Wilm, Maxime Ulysse Garcia, Paolo Di Tommaso & Sven Nahnsen. + > + > _Nat Biotechnol._ 2020 Feb 13. doi: [10.1038/s41587-020-0439-x](https://dx.doi.org/10.1038/s41587-020-0439-x). - Sample Description """ - pvc_name: str = initialize() + pvc_name: str = initialize(run_name=run_name) nextflow_runtime( pvc_name=pvc_name, input=input, @@ -240,12 +339,12 @@ def nf_nf_core_cutandrun( "blacklist": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-blacklist.bed"), "spikein_bowtie2": LatchFile("s3://latch-public/test-data/35729/cutandrun/e_coli_U00096_3.tar.gz"), "spikein_fasta": LatchFile("s3://latch-public/test-data/35729/cutandrun/e_coli_U00096_3.fa"), + "genome_source": "input_ref", "fasta": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20.fa"), "bowtie2": LatchDir("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-bowtie2/"), "gtf": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-genes.gtf"), "gene_bed": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-chr20-genes.bed"), - "run_name": "test-run", - "outdir": LatchOutputDir("latch:///cutandrun_test/"), + "run_name": "Test_Run", "peakcaller": "seacr", "mito_name": "chrM", "remove_linear_duplicates": True, diff --git a/wf/entrypoint.py b/wf/entrypoint.py index f439fdd2..9a3052ec 100644 --- a/wf/entrypoint.py +++ b/wf/entrypoint.py @@ -2,18 +2,15 @@ import shutil import subprocess import sys -import typing from dataclasses import dataclass from enum import Enum from pathlib import Path +from typing import List, Optional import requests -import typing_extensions -from flytekit.core.annotation import FlyteAnnotation -from latch.executions import report_nextflow_used_storage +from latch.executions import rename_current_execution, report_nextflow_used_storage from latch.ldata.path import LPath from latch.resources.tasks import custom_task, nextflow_runtime_task -from latch.resources.workflow import workflow from latch.types import metadata from latch.types.directory import LatchDir, LatchOutputDir from latch.types.file import LatchFile @@ -22,13 +19,16 @@ from latch_cli.services.register.utils import import_module_by_path from latch_cli.utils import urljoins +sys.stdout.reconfigure(line_buffering=True) + meta = Path("latch_metadata") / "__init__.py" import_module_by_path(meta) -import latch_metadata @custom_task(cpu=0.25, memory=0.5, storage_gib=1) -def initialize() -> str: +def initialize(run_name: str) -> str: + rename_current_execution(str(run_name)) + token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID") if token is None: raise RuntimeError("failed to get execution token") @@ -39,7 +39,10 @@ def initialize() -> str: resp = requests.post( "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage", headers=headers, - json={"storage_expiration_hours": 0, "version": 2}, + json={ + "storage_expiration_hours": 0, + "version": 2, + }, ) resp.raise_for_status() print("Done.") @@ -57,8 +60,8 @@ class SampleSheet: class Reference(Enum): - hg19 = "GRCh37 (Homo Sapiens hg19)" hg38 = "GRCh38 (Homo Sapiens hg38)" + hg19 = "GRCh37 (Homo Sapiens hg19)" input_construct_samplesheet = metadata._nextflow_metadata.parameters["input"].samplesheet_constructor @@ -67,27 +70,27 @@ class Reference(Enum): @nextflow_runtime_task(cpu=4, memory=8, storage_gib=100) def nextflow_runtime( pvc_name: str, - input: typing.List[SampleSheet], + input: List[SampleSheet], outdir: LatchOutputDir, - multiqc_title: typing.Optional[str], + multiqc_title: Optional[str], save_reference: bool, save_merged_fastq: bool, save_trimmed: bool, save_spikein_aligned: bool, save_unaligned: bool, save_align_intermed: bool, - email: typing.Optional[str], + email: Optional[str], dump_scale_factors: bool, genome_source: str, - genome: typing.Optional[str], - latch_genome: typing.Optional[Reference], - bowtie2: typing.Optional[LatchDir], - gtf: typing.Optional[LatchFile], - gene_bed: typing.Optional[LatchFile], - blacklist: typing.Optional[LatchFile], - spikein_bowtie2: typing.Optional[LatchFile], - spikein_fasta: typing.Optional[LatchFile], - fasta: typing.Optional[LatchFile], + genome: Optional[str], + latch_genome: Optional[Reference], + bowtie2: Optional[LatchDir], + gtf: Optional[LatchFile], + gene_bed: Optional[LatchFile], + blacklist: Optional[LatchFile], + spikein_bowtie2: Optional[LatchFile], + spikein_fasta: Optional[LatchFile], + fasta: Optional[LatchFile], only_input: bool, only_genome: bool, only_preqc: bool, @@ -106,46 +109,46 @@ def nextflow_runtime( skip_multiqc: bool, remove_mitochondrial_reads: bool, run_name: str, - mito_name: typing.Optional[str], + mito_name: Optional[str], dedup_target_reads: bool, remove_linear_duplicates: bool, - macs2_pvalue: typing.Optional[float], + macs2_pvalue: Optional[float], singularity_pull_docker_container: bool, - multiqc_methods_description: typing.Optional[str], - spikein_genome: typing.Optional[str], - clip_r1: typing.Optional[int], - clip_r2: typing.Optional[int], - three_prime_clip_r1: typing.Optional[int], - three_prime_clip_r2: typing.Optional[int], - trim_nextseq: typing.Optional[int], - minimum_alignment_q_score: typing.Optional[int], + multiqc_methods_description: Optional[str], + spikein_genome: Optional[str], + clip_r1: Optional[int], + clip_r2: Optional[int], + three_prime_clip_r1: Optional[int], + three_prime_clip_r2: Optional[int], + trim_nextseq: Optional[int], + minimum_alignment_q_score: Optional[int], end_to_end: bool, - normalisation_mode: typing.Optional[str], - normalisation_binsize: typing.Optional[int], - peakcaller: typing.Optional[str], + normalisation_mode: Optional[str], + normalisation_binsize: Optional[int], + peakcaller: Optional[str], use_control: bool, extend_fragments: bool, - igg_scale_factor: typing.Optional[float], - seacr_peak_threshold: typing.Optional[float], - seacr_norm: typing.Optional[str], - seacr_stringent: typing.Optional[str], - macs2_qvalue: typing.Optional[float], - macs_gsize: typing.Optional[float], + igg_scale_factor: Optional[float], + seacr_peak_threshold: Optional[float], + seacr_norm: Optional[str], + seacr_stringent: Optional[str], + macs2_qvalue: Optional[float], + macs_gsize: Optional[float], macs2_narrow_peak: bool, - macs2_broad_cutoff: typing.Optional[float], - consensus_peak_mode: typing.Optional[str], - replicate_threshold: typing.Optional[float], + macs2_broad_cutoff: Optional[float], + consensus_peak_mode: Optional[str], + replicate_threshold: Optional[float], igv_show_gene_names: bool, - dt_qc_bam_binsize: typing.Optional[int], - dt_qc_corr_method: typing.Optional[str], - dt_heatmap_gene_beforelen: typing.Optional[int], - dt_heatmap_gene_bodylen: typing.Optional[int], - dt_heatmap_gene_afterlen: typing.Optional[int], - dt_heatmap_peak_beforelen: typing.Optional[int], - dt_heatmap_peak_afterlen: typing.Optional[int], + dt_qc_bam_binsize: Optional[int], + dt_qc_corr_method: Optional[str], + dt_heatmap_gene_beforelen: Optional[int], + dt_heatmap_gene_bodylen: Optional[int], + dt_heatmap_gene_afterlen: Optional[int], + dt_heatmap_peak_beforelen: Optional[int], + dt_heatmap_peak_afterlen: Optional[int], dt_calc_all_matrix: bool, - min_frip_overlap: typing.Optional[float], - min_peak_overlap: typing.Optional[float], + min_frip_overlap: Optional[float], + min_peak_overlap: Optional[float], igv_sort_by_groups: bool, ) -> None: shared_dir = Path("/nf-workdir") @@ -173,14 +176,6 @@ def nextflow_runtime( dirs_exist_ok=True, ) - profile_list = ["docker"] - if False: - profile_list.extend([p.value for p in execution_profiles]) - - if len(profile_list) == 0: - profile_list.append("standard") - - profiles = ",".join(profile_list) if genome_source == "latch_genome_source": fasta = os.path.join( "s3://latch-public/test-data/35729/cutandrun/", @@ -201,7 +196,7 @@ def nextflow_runtime( "-work-dir", str(shared_dir), "-profile", - profiles, + "docker", "-c", "latch.config", "-resume", From 0f3290b1fc6f5fbf83394a811ffb1c2bf116529c Mon Sep 17 00:00:00 2001 From: Tahir D'Mello Date: Mon, 30 Sep 2024 20:58:31 -0700 Subject: [PATCH 5/5] dispatcher + ofs update --- .latch/Dockerfile | 2 +- .latch/config | 2 +- latch_metadata/__init__.py | 1 + latch_metadata/parameters.py | 91 ++++++++++++++++++------------------ wf/__init__.py | 4 +- wf/entrypoint.py | 6 +-- 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/.latch/Dockerfile b/.latch/Dockerfile index 0f308532..49f6c543 100644 --- a/.latch/Dockerfile +++ b/.latch/Dockerfile @@ -1,5 +1,5 @@ # DO NOT CHANGE -from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0 +from 812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.1.2 workdir /tmp/docker-build/work/ diff --git a/.latch/config b/.latch/config index 96f56f3f..14a68ec5 100644 --- a/.latch/config +++ b/.latch/config @@ -1 +1 @@ -{"latch_version": "2.52.1", "base_image": "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.0.0", "date": "2024-09-12T12:57:15.404807"} +{"latch_version": "2.52.1", "base_image": "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base-nextflow:v2.1.2", "date": "2024-09-12T12:57:15.404807"} diff --git a/latch_metadata/__init__.py b/latch_metadata/__init__.py index f39b7ffa..bad6f293 100644 --- a/latch_metadata/__init__.py +++ b/latch_metadata/__init__.py @@ -17,6 +17,7 @@ Section( "Input", Params("input"), + Text("Samplesheet expects 'control' column to be filled for all non-control samples."), Spoiler( "Optional", Params("use_control"), diff --git a/latch_metadata/parameters.py b/latch_metadata/parameters.py index 180076f7..63b18f04 100644 --- a/latch_metadata/parameters.py +++ b/latch_metadata/parameters.py @@ -1,6 +1,6 @@ -import typing from dataclasses import dataclass from enum import Enum +from typing import List, Optional from latch.types.directory import LatchDir, LatchOutputDir from latch.types.file import LatchFile @@ -13,7 +13,7 @@ class SampleSheet: replicate: int fastq_1: LatchFile fastq_2: LatchFile - control: str + control: Optional[str] class Reference(Enum): @@ -23,7 +23,7 @@ class Reference(Enum): generated_parameters = { "input": NextflowParameter( - type=typing.List[SampleSheet], + type=List[SampleSheet], display_name="Input", description="Information about the samples in the experiment", section_title=None, @@ -38,7 +38,7 @@ class Reference(Enum): description="The output directory where the results will be saved. You have to use absolute paths to store on Cloud infrastructure.", ), "multiqc_title": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default=None, section_title=None, display_name="MultiQC Title", @@ -64,9 +64,8 @@ class Reference(Enum): description="Choose Reference Genome", ), "latch_genome": NextflowParameter( - type=typing.Optional[Reference], + type=Optional[Reference], default=None, - section_title="Reference genome options", display_name="Genome", description="Name of the Latch Verfied Reference Genome", ), @@ -99,7 +98,7 @@ class Reference(Enum): description="Save alignment intermediates to the output directory (WARNING: can be very large)", ), "email": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default=None, section_title=None, display_name="Email", @@ -113,63 +112,63 @@ class Reference(Enum): description="Output calculated scale factors from pipeline", ), "genome": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default=None, section_title=None, display_name="Genome", description="Name of iGenomes reference.", ), "bowtie2": NextflowParameter( - type=typing.Optional[LatchDir], + type=Optional[LatchDir], default=None, section_title=None, display_name="Bowtie2", description="Path to bowtie2 index", ), "gtf": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="GTF", description="Path to GTF annotation file", ), "gene_bed": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="Gene Bed", description="Path to gene BED file", ), "blacklist": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="Blacklist", description="Path to genome blacklist", ), "spikein_genome": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="K12-MG1655", section_title=None, display_name="Spikein Genome", description="Name of the iGenome reference for the spike-in genome, defaulting to E. coli K12, for yeast set to R64-1-1, for fruit fly BDGP6", ), "spikein_bowtie2": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="Spikein Bowtie2", description="Path to spike-in bowtie2 index", ), "spikein_fasta": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="Spikein Fasta", description="Path to spike-in fasta", ), "fasta": NextflowParameter( - type=typing.Optional[LatchFile], + type=Optional[LatchFile], default=None, section_title=None, display_name="Fasta", @@ -288,42 +287,42 @@ class Reference(Enum): description="Skips multiqc", ), "clip_r1": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=0, section_title=None, display_name="Clip R1", description="Instructs Trim Galore to remove bp from the 5' end of read 1 (or single-end reads).", ), "clip_r2": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=0, section_title=None, display_name="Clip R2", description="Instructs Trim Galore to remove bp from the 5' end of read 2 (paired-end reads only).", ), "three_prime_clip_r1": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=0, section_title=None, display_name="Three Prime Clip R1", description="Instructs Trim Galore to remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed.", ), "three_prime_clip_r2": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=0, section_title=None, display_name="Three Prime Clip R2", description="Instructs Trim Galore to remove bp from the 3' end of read 2 AFTER adapter/quality trimming has been performed.", ), "trim_nextseq": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=0, section_title=None, display_name="Trim NextSeq", description="Instructs Trim Galore to apply the --nextseq=X option, to trim based on quality after removing poly-G tails.", ), "minimum_alignment_q_score": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=20, section_title=None, display_name="Minimum Alignment Q Score", @@ -349,7 +348,7 @@ class Reference(Enum): ], ), "mito_name": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default=None, section_title=None, display_name="Mito Name", @@ -377,7 +376,7 @@ class Reference(Enum): description="Use --end-to-end mode of Bowtie2 during alignment", ), "normalisation_mode": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="Spikein", section_title=None, display_name="Normalisation Mode", @@ -385,14 +384,14 @@ class Reference(Enum): description='Sets the target read normalisation mode. Options are: ["Spikein", "RPKM", "CPM", "BPM", "None" ]', ), "normalisation_binsize": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=50, section_title=None, display_name="Normalisation Binsize", description='If normalisation option is one of "RPKM", "CPM", "BPM" - then the binsize that the reads count is calculated on is used.', ), "peakcaller": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="seacr", section_title=None, display_name="Peakcaller", @@ -414,21 +413,21 @@ class Reference(Enum): description="Specifies whether to extend paired-end fragments between the read mates when calculating coverage tracks", ), "igg_scale_factor": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.5, section_title=None, display_name="IGG Scale Factor", description="Specifies whether the background control is scaled prior to being used to normalise peaks.", ), "seacr_peak_threshold": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.05, section_title=None, display_name="SEACR Peak Threshold", description="SEACR specifies returns the top n fraction (between 0 and 1) of peaks based on total signal within peaks. This is only used if there are no controls included with the samples and if `--use_control` is `false`", ), "seacr_norm": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="non", section_title=None, appearance_type=Multiselect(["non", "norm"], allow_custom=False), @@ -436,7 +435,7 @@ class Reference(Enum): description="SEACR normalization.", ), "seacr_stringent": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="stringent", section_title=None, appearance_type=Multiselect(["stringent", "relaxed"], allow_custom=False), @@ -444,21 +443,21 @@ class Reference(Enum): description="SEACR stringency.", ), "macs2_qvalue": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.01, section_title=None, display_name="MACS2 Q-value", description="Q-value threshold for MACS2 peak caller.", ), "macs2_pvalue": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=None, section_title=None, display_name="MACS2 P-value", description="P-value threshold for macs2 peak caller. If set it will override the q-value.", ), "macs_gsize": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=2700000000.0, section_title=None, display_name="MACS Gsize", @@ -472,14 +471,14 @@ class Reference(Enum): description="Determines whether MACS2 broad or narrow peak mode is used for the peak caller", ), "macs2_broad_cutoff": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.1, section_title=None, display_name="MACS2 Broad Cutoff", description="MACS2 broad cutoff parameter", ), "consensus_peak_mode": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="group", section_title=None, display_name="Consensus Peak Mode", @@ -487,7 +486,7 @@ class Reference(Enum): description="Specifies what samples to group together for consensus peaks. Options are [group, all]", ), "replicate_threshold": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=1.0, section_title=None, display_name="Replicate Threshold", @@ -501,49 +500,49 @@ class Reference(Enum): description="Show gene names instead of symbols in IGV browser sessions", ), "dt_qc_bam_binsize": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=500, section_title=None, display_name="DT QC BAM Binsize", description="Deeptools multiBamSummary bam bin size", ), "dt_qc_corr_method": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default="pearson", section_title=None, display_name="DT QC Corr Method", description="Deeptools Correlation Plot statistical calculation method", ), "dt_heatmap_gene_beforelen": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=3000, section_title=None, display_name="DT Heatmap Gene Beforelen", description="Deeptools heatmap gene plot before length (bases)", ), "dt_heatmap_gene_bodylen": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=5000, section_title=None, display_name="DT Heatmap Gene Bodylen", description="Deeptools heatmap gene plot body length (bases)", ), "dt_heatmap_gene_afterlen": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=3000, section_title=None, display_name="DT Heatmap Gene Afterlen", description="Deeptools heatmap gene plot after length (bases)", ), "dt_heatmap_peak_beforelen": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=3000, section_title=None, display_name="DT Heatmap Peak Beforelen", description="Deeptools heatmap peak plot before length (bases)", ), "dt_heatmap_peak_afterlen": NextflowParameter( - type=typing.Optional[int], + type=Optional[int], default=3000, section_title=None, display_name="DT Heatmap Peak Afterlen", @@ -557,14 +556,14 @@ class Reference(Enum): description="Flag for whether to generate a heatmap for all samples together", ), "min_frip_overlap": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.2, section_title=None, display_name="Min FRiP Overlap", description="Minimum fragment overlap for FriP score", ), "min_peak_overlap": NextflowParameter( - type=typing.Optional[float], + type=Optional[float], default=0.2, section_title=None, display_name="Min Peak Overlap", @@ -585,7 +584,7 @@ class Reference(Enum): description="Pull Docker container.", ), "multiqc_methods_description": NextflowParameter( - type=typing.Optional[str], + type=Optional[str], default=None, section_title=None, display_name="MultiQC Methods Description", diff --git a/wf/__init__.py b/wf/__init__.py index b8e7c822..cee09a9e 100644 --- a/wf/__init__.py +++ b/wf/__init__.py @@ -326,14 +326,14 @@ def nf_nf_core_cutandrun( replicate=1, fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep1_r1.fastq.gz"), fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep1_r2.fastq.gz"), - control="", + control=None, ), SampleSheet( group="igg", replicate=2, fastq_1=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep2_r1.fastq.gz"), fastq_2=LatchFile("s3://latch-public/test-data/35729/cutandrun/test_data/igg_rep2_r2.fastq.gz"), - control="", + control=None, ), ], "blacklist": LatchFile("s3://latch-public/test-data/35729/cutandrun/hg38-blacklist.bed"), diff --git a/wf/entrypoint.py b/wf/entrypoint.py index 9a3052ec..a2fd7e24 100644 --- a/wf/entrypoint.py +++ b/wf/entrypoint.py @@ -37,10 +37,10 @@ def initialize(run_name: str) -> str: print("Provisioning shared storage volume... ", end="") resp = requests.post( - "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage", + "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage-ofs", headers=headers, json={ - "storage_expiration_hours": 0, + "storage_expiration_hours": 1, "version": 2, }, ) @@ -56,7 +56,7 @@ class SampleSheet: replicate: int fastq_1: LatchFile fastq_2: LatchFile - control: str + control: Optional[str] class Reference(Enum):