CI
15cd1e1fa3
Build pipeline: viash-hub.htrnaseq.main-pipeline-ls5l5
Source commit: bdd7091d5b
Source message: Fix params.yaml and update integration_test
1626 lines
62 KiB
Bash
Executable File
1626 lines
62 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# parallel_map main_pipeline
|
|
#
|
|
# This wrapper script is auto-generated by viash 0.9.0-RC6 and is thus a
|
|
# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from
|
|
# Data Intuitive.
|
|
#
|
|
# The component may contain files which fall under a different license. The
|
|
# authors of this component should specify the license in the header of such
|
|
# files, or include a separate license file detailing the licenses of all included
|
|
# files.
|
|
|
|
set -e
|
|
|
|
if [ -z "$VIASH_TEMP" ]; then
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TEMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-/tmp}
|
|
fi
|
|
|
|
# define helper functions
|
|
# ViashQuote: put quotes around non flag values
|
|
# $1 : unquoted string
|
|
# return : possibly quoted string
|
|
# examples:
|
|
# ViashQuote --foo # returns --foo
|
|
# ViashQuote bar # returns 'bar'
|
|
# Viashquote --foo=bar # returns --foo='bar'
|
|
function ViashQuote {
|
|
if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then
|
|
echo "$1" | sed "s#=\(.*\)#='\1'#"
|
|
elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then
|
|
echo "$1"
|
|
else
|
|
echo "'$1'"
|
|
fi
|
|
}
|
|
# ViashRemoveFlags: Remove leading flag
|
|
# $1 : string with a possible leading flag
|
|
# return : string without possible leading flag
|
|
# examples:
|
|
# ViashRemoveFlags --foo=bar # returns bar
|
|
function ViashRemoveFlags {
|
|
echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//'
|
|
}
|
|
# ViashSourceDir: return the path of a bash file, following symlinks
|
|
# usage : ViashSourceDir ${BASH_SOURCE[0]}
|
|
# $1 : Should always be set to ${BASH_SOURCE[0]}
|
|
# returns : The absolute path of the bash file
|
|
function ViashSourceDir {
|
|
SOURCE="$1"
|
|
while [ -h "$SOURCE" ]; do
|
|
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
|
|
SOURCE="$(readlink "$SOURCE")"
|
|
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
|
|
done
|
|
cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd
|
|
}
|
|
# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks
|
|
# usage : ViashFindTargetDir 'ScriptPath'
|
|
# $1 : The location from where to start the upward search
|
|
# returns : The absolute path of the '.build.yaml' file
|
|
function ViashFindTargetDir {
|
|
SOURCE="$1"
|
|
while [[ "$SOURCE" != "" && ! -e "$SOURCE/.build.yaml" ]]; do
|
|
SOURCE=${SOURCE%/*}
|
|
done
|
|
echo $SOURCE
|
|
}
|
|
# see https://en.wikipedia.org/wiki/Syslog#Severity_level
|
|
VIASH_LOGCODE_EMERGENCY=0
|
|
VIASH_LOGCODE_ALERT=1
|
|
VIASH_LOGCODE_CRITICAL=2
|
|
VIASH_LOGCODE_ERROR=3
|
|
VIASH_LOGCODE_WARNING=4
|
|
VIASH_LOGCODE_NOTICE=5
|
|
VIASH_LOGCODE_INFO=6
|
|
VIASH_LOGCODE_DEBUG=7
|
|
VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE
|
|
|
|
# ViashLog: Log events depending on the verbosity level
|
|
# usage: ViashLog 1 alert Oh no something went wrong!
|
|
# $1: required verbosity level
|
|
# $2: display tag
|
|
# $3+: messages to display
|
|
# stdout: Your input, prepended by '[$2] '.
|
|
function ViashLog {
|
|
local required_level="$1"
|
|
local display_tag="$2"
|
|
shift 2
|
|
if [ $VIASH_VERBOSITY -ge $required_level ]; then
|
|
>&2 echo "[$display_tag]" "$@"
|
|
fi
|
|
}
|
|
|
|
# ViashEmergency: log events when the system is unstable
|
|
# usage: ViashEmergency Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[emergency] '.
|
|
function ViashEmergency {
|
|
ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@"
|
|
}
|
|
|
|
# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database)
|
|
# usage: ViashAlert Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[alert] '.
|
|
function ViashAlert {
|
|
ViashLog $VIASH_LOGCODE_ALERT alert "$@"
|
|
}
|
|
|
|
# ViashCritical: log events when a critical condition occurs
|
|
# usage: ViashCritical Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[critical] '.
|
|
function ViashCritical {
|
|
ViashLog $VIASH_LOGCODE_CRITICAL critical "$@"
|
|
}
|
|
|
|
# ViashError: log events when an error condition occurs
|
|
# usage: ViashError Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[error] '.
|
|
function ViashError {
|
|
ViashLog $VIASH_LOGCODE_ERROR error "$@"
|
|
}
|
|
|
|
# ViashWarning: log potentially abnormal events
|
|
# usage: ViashWarning Something may have gone wrong.
|
|
# stdout: Your input, prepended by '[warning] '.
|
|
function ViashWarning {
|
|
ViashLog $VIASH_LOGCODE_WARNING warning "$@"
|
|
}
|
|
|
|
# ViashNotice: log significant but normal events
|
|
# usage: ViashNotice This just happened.
|
|
# stdout: Your input, prepended by '[notice] '.
|
|
function ViashNotice {
|
|
ViashLog $VIASH_LOGCODE_NOTICE notice "$@"
|
|
}
|
|
|
|
# ViashInfo: log normal events
|
|
# usage: ViashInfo This just happened.
|
|
# stdout: Your input, prepended by '[info] '.
|
|
function ViashInfo {
|
|
ViashLog $VIASH_LOGCODE_INFO info "$@"
|
|
}
|
|
|
|
# ViashDebug: log all events, for debugging purposes
|
|
# usage: ViashDebug This just happened.
|
|
# stdout: Your input, prepended by '[debug] '.
|
|
function ViashDebug {
|
|
ViashLog $VIASH_LOGCODE_DEBUG debug "$@"
|
|
}
|
|
|
|
# find source folder of this component
|
|
VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}`
|
|
|
|
# find the root of the built components & dependencies
|
|
VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR`
|
|
|
|
# define meta fields
|
|
VIASH_META_NAME="parallel_map"
|
|
VIASH_META_FUNCTIONALITY_NAME="parallel_map"
|
|
VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME"
|
|
VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
|
|
VIASH_META_TEMP_DIR="$VIASH_TEMP"
|
|
|
|
|
|
# ViashHelp: Display helpful explanation about this executable
|
|
function ViashHelp {
|
|
echo "parallel_map main_pipeline"
|
|
echo ""
|
|
echo "Map wells in batch, using STAR"
|
|
echo "Spliced Transcripts Alignment to a Reference (C) Alexander Dobin"
|
|
echo "https://github.com/alexdobin/STAR"
|
|
echo ""
|
|
echo "Input arguments:"
|
|
echo " --input_r1"
|
|
echo " type: file, required parameter, multiple values allowed, file must exist"
|
|
echo ""
|
|
echo " --input_r2"
|
|
echo " type: file, required parameter, multiple values allowed, file must exist"
|
|
echo ""
|
|
echo " --genomeDir"
|
|
echo " type: file, required parameter, file must exist"
|
|
echo " STAR reference directory"
|
|
echo ""
|
|
echo " --barcodes"
|
|
echo " type: string, required parameter, multiple values allowed"
|
|
echo " The barcodes/wells to process"
|
|
echo ""
|
|
echo "Barcode arguments:"
|
|
echo " --wellBarcodesLength"
|
|
echo " type: integer, required parameter"
|
|
echo " The length of the well barcodes"
|
|
echo ""
|
|
echo " --umiLength"
|
|
echo " type: integer, required parameter"
|
|
echo " The length of the UMIs"
|
|
echo ""
|
|
echo " --limitBAMsortRAM"
|
|
echo " type: string"
|
|
echo " default: 10000000000"
|
|
echo ""
|
|
echo "Runtime arguments:"
|
|
echo " --runThreadN"
|
|
echo " type: integer"
|
|
echo " default: 1"
|
|
echo " Number of threads to use for a single STAR execution."
|
|
echo ""
|
|
echo "Output arguments:"
|
|
echo " --output"
|
|
echo " type: file, required parameter, multiple values allowed, output, file"
|
|
echo "must exist"
|
|
echo " default: ./*"
|
|
echo " Location of the output folders, 1 folder per barcode. The value used"
|
|
echo " for this argument must contain a '*', which will be replaced with the"
|
|
echo " barcode to form the final output location for that barcode."
|
|
echo ""
|
|
echo " --joblog"
|
|
echo " type: file, output, file must exist"
|
|
echo " default: execution_log.txt"
|
|
echo " Where to store the log file listing all the jobs."
|
|
}
|
|
|
|
# initialise variables
|
|
VIASH_MODE='run'
|
|
VIASH_ENGINE_ID='docker'
|
|
|
|
######## Helper functions for setting up Docker images for viash ########
|
|
# expects: ViashDockerBuild
|
|
|
|
# ViashDockerInstallationCheck: check whether Docker is installed correctly
|
|
#
|
|
# examples:
|
|
# ViashDockerInstallationCheck
|
|
function ViashDockerInstallationCheck {
|
|
ViashDebug "Checking whether Docker is installed"
|
|
if [ ! command -v docker &> /dev/null ]; then
|
|
ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions."
|
|
exit 1
|
|
fi
|
|
|
|
ViashDebug "Checking whether the Docker daemon is running"
|
|
save=$-; set +e
|
|
docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null)
|
|
out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashCritical "Docker daemon does not seem to be running. Try one of the following:"
|
|
ViashCritical "- Try running 'dockerd' in the command line"
|
|
ViashCritical "- See https://docs.docker.com/config/daemon/"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerRemoteTagCheck: check whether a Docker image is available
|
|
# on a remote. Assumes `docker login` has been performed, if relevant.
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerRemoteTagCheck python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerRemoteTagCheck sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerRemoteTagCheck {
|
|
docker manifest inspect $1 > /dev/null 2> /dev/null
|
|
}
|
|
|
|
# ViashDockerLocalTagCheck: check whether a Docker image is available locally
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# docker pull python:latest
|
|
# ViashDockerLocalTagCheck python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerLocalTagCheck sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerLocalTagCheck {
|
|
[ -n "$(docker images -q $1)" ]
|
|
}
|
|
|
|
# ViashDockerPull: pull a Docker image
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerPull python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerPull sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerPull {
|
|
ViashNotice "Checking if Docker image is available at '$1'"
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
docker pull $1 && return 0 || return 1
|
|
else
|
|
save=$-; set +e
|
|
docker pull $1 2> /dev/null > /dev/null
|
|
out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible."
|
|
fi
|
|
return $out
|
|
fi
|
|
}
|
|
|
|
# ViashDockerPush: push a Docker image
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerPush python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerPush sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerPush {
|
|
ViashNotice "Pushing image to '$1'"
|
|
save=$-; set +e
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
docker push $1
|
|
out=$?
|
|
else
|
|
docker push $1 2> /dev/null > /dev/null
|
|
out=$?
|
|
fi
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -eq 0 ]; then
|
|
ViashNotice "Container '$1' push succeeded."
|
|
else
|
|
ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions."
|
|
fi
|
|
return $out
|
|
}
|
|
|
|
# ViashDockerPullElseBuild: pull a Docker image, else build it
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument.
|
|
# examples:
|
|
# ViashDockerPullElseBuild mynewcomponent
|
|
function ViashDockerPullElseBuild {
|
|
save=$-; set +e
|
|
ViashDockerPull $1
|
|
out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashDockerBuild $@
|
|
fi
|
|
}
|
|
|
|
# ViashDockerSetup: create a Docker image, according to specified docker setup strategy
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $2 : docker setup strategy, see DockerSetupStrategy.scala
|
|
# examples:
|
|
# ViashDockerSetup mynewcomponent alwaysbuild
|
|
function ViashDockerSetup {
|
|
local image_id="$1"
|
|
local setup_strategy="$2"
|
|
if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then
|
|
ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then
|
|
ViashDockerPull $image_id
|
|
elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then
|
|
ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then
|
|
ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then
|
|
ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then
|
|
local save=$-; set +e
|
|
ViashDockerLocalTagCheck $image_id
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -eq 0 ]; then
|
|
ViashInfo "Image $image_id already exists"
|
|
elif [ "$setup_strategy" == "ifneedbebuild" ]; then
|
|
ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then
|
|
ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbepull" ]; then
|
|
ViashDockerPull $image_id
|
|
elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then
|
|
ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then
|
|
ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
else
|
|
ViashError "Unrecognised Docker strategy: $setup_strategy"
|
|
exit 1
|
|
fi
|
|
elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then
|
|
ViashDockerPush "$image_id"
|
|
elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then
|
|
local save=$-; set +e
|
|
ViashDockerRemoteTagCheck $image_id
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -eq 0 ]; then
|
|
ViashNotice "Container '$image_id' exists, doing nothing."
|
|
else
|
|
ViashNotice "Container '$image_id' does not yet exist."
|
|
ViashDockerPush "$image_id"
|
|
fi
|
|
elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then
|
|
ViashNotice "Skipping setup."
|
|
else
|
|
ViashError "Unrecognised Docker strategy: $setup_strategy"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerCheckCommands: Check whether a docker container has the required commands
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $@ : commands to verify being present
|
|
# examples:
|
|
# ViashDockerCheckCommands bash:4.0 bash ps foo
|
|
function ViashDockerCheckCommands {
|
|
local image_id="$1"
|
|
shift 1
|
|
local commands="$@"
|
|
local save=$-; set +e
|
|
local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0'
|
|
missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done")
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -ne 0 ]; then
|
|
ViashError "Docker container '$image_id' does not contain command '$missing'."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerBuild: build a docker image
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $... : additional arguments to pass to docker build
|
|
# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in
|
|
# $VIASH_META_NAME : name of the component
|
|
# $VIASH_META_RESOURCES_DIR : directory containing the resources
|
|
# $VIASH_VERBOSITY : verbosity level
|
|
# exit code $? : whether or not the image was built successfully
|
|
function ViashDockerBuild {
|
|
local image_id="$1"
|
|
shift 1
|
|
|
|
# create temporary directory to store dockerfile & optional resources in
|
|
local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX")
|
|
local dockerfile="$tmpdir/Dockerfile"
|
|
function clean_up {
|
|
rm -rf "$tmpdir"
|
|
}
|
|
trap clean_up EXIT
|
|
|
|
# store dockerfile and resources
|
|
ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile"
|
|
|
|
# generate the build command
|
|
local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'"
|
|
|
|
# build the container
|
|
ViashNotice "Building container '$image_id' with Dockerfile"
|
|
ViashInfo "$docker_build_cmd"
|
|
local save=$-; set +e
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
eval $docker_build_cmd
|
|
else
|
|
eval $docker_build_cmd &> "$tmpdir/docker_build.log"
|
|
fi
|
|
|
|
# check exit code
|
|
local out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashError "Error occurred while building container '$image_id'"
|
|
if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then
|
|
ViashError "Transcript: --------------------------------"
|
|
cat "$tmpdir/docker_build.log"
|
|
ViashError "End of transcript --------------------------"
|
|
fi
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
######## End of helper functions for setting up Docker images for viash ########
|
|
|
|
# ViashDockerFile: print the dockerfile to stdout
|
|
# $1 : engine identifier
|
|
# return : dockerfile required to run this component
|
|
# examples:
|
|
# ViashDockerFile
|
|
function ViashDockerfile {
|
|
local engine_id="$1"
|
|
|
|
if [[ "$engine_id" == "docker" ]]; then
|
|
cat << 'VIASHDOCKER'
|
|
FROM debian:stable-slim
|
|
ENTRYPOINT []
|
|
RUN apt-get update && \
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y procps gzip bzip2 parallel wget zlib1g-dev unzip xxd file && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
ENV STAR_VERSION "2.7.11b"
|
|
ENV STAR_SOURCE "https://github.com/alexdobin/STAR/releases/download/$STAR_VERSION/STAR_$STAR_VERSION.zip"
|
|
ENV STAR_TARGET "/tmp/star.zip"
|
|
ENV STAR_BINARY "STAR"
|
|
RUN wget -O $STAR_TARGET $STAR_SOURCE && \
|
|
unzip $STAR_TARGET -d /tmp && \
|
|
mv /tmp/STAR_$STAR_VERSION/Linux_x86_64_static/STAR /usr/local/bin/$STAR_BINARY && \
|
|
chmod +x /usr/local/bin/$STAR_BINARY && \
|
|
rm $STAR_TARGET && rm -rf /tmp/STAR_$STAR_VERSION
|
|
|
|
LABEL org.opencontainers.image.description="Companion container for running component parallel_map"
|
|
LABEL org.opencontainers.image.created="2024-08-20T15:27:42Z"
|
|
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
|
|
LABEL org.opencontainers.image.revision="bdd7091d5b5982b84103a43fece50c608810e0aa"
|
|
LABEL org.opencontainers.image.version="main_pipeline"
|
|
|
|
VIASHDOCKER
|
|
fi
|
|
}
|
|
|
|
# ViashDockerBuildArgs: return the arguments to pass to docker build
|
|
# $1 : engine identifier
|
|
# return : arguments to pass to docker build
|
|
function ViashDockerBuildArgs {
|
|
local engine_id="$1"
|
|
|
|
if [[ "$engine_id" == "docker" ]]; then
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# ViashAbsolutePath: generate absolute path from relative path
|
|
# borrowed from https://stackoverflow.com/a/21951256
|
|
# $1 : relative filename
|
|
# return : absolute path
|
|
# examples:
|
|
# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt
|
|
# ViashAbsolutePath /foo/bar/.. # returns /foo
|
|
function ViashAbsolutePath {
|
|
local thePath
|
|
if [[ ! "$1" =~ ^/ ]]; then
|
|
thePath="$PWD/$1"
|
|
else
|
|
thePath="$1"
|
|
fi
|
|
echo "$thePath" | (
|
|
IFS=/
|
|
read -a parr
|
|
declare -a outp
|
|
for i in "${parr[@]}"; do
|
|
case "$i" in
|
|
''|.) continue ;;
|
|
..)
|
|
len=${#outp[@]}
|
|
if ((len==0)); then
|
|
continue
|
|
else
|
|
unset outp[$((len-1))]
|
|
fi
|
|
;;
|
|
*)
|
|
len=${#outp[@]}
|
|
outp[$len]="$i"
|
|
;;
|
|
esac
|
|
done
|
|
echo /"${outp[*]}"
|
|
)
|
|
}
|
|
# ViashDockerAutodetectMount: auto configuring docker mounts from parameters
|
|
# $1 : The parameter value
|
|
# returns : New parameter
|
|
# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker
|
|
# examples:
|
|
# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar'
|
|
# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"'
|
|
function ViashDockerAutodetectMount {
|
|
abs_path=$(ViashAbsolutePath "$1")
|
|
if [ -d "$abs_path" ]; then
|
|
mount_source="$abs_path"
|
|
base_name=""
|
|
else
|
|
mount_source=`dirname "$abs_path"`
|
|
base_name=`basename "$abs_path"`
|
|
fi
|
|
mount_target="/viash_automount$mount_source"
|
|
if [ -z "$base_name" ]; then
|
|
echo "$mount_target"
|
|
else
|
|
echo "$mount_target/$base_name"
|
|
fi
|
|
}
|
|
function ViashDockerAutodetectMountArg {
|
|
abs_path=$(ViashAbsolutePath "$1")
|
|
if [ -d "$abs_path" ]; then
|
|
mount_source="$abs_path"
|
|
base_name=""
|
|
else
|
|
mount_source=`dirname "$abs_path"`
|
|
base_name=`basename "$abs_path"`
|
|
fi
|
|
mount_target="/viash_automount$mount_source"
|
|
ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target"
|
|
echo "--volume=\"$mount_source:$mount_target\""
|
|
}
|
|
function ViashDockerStripAutomount {
|
|
abs_path=$(ViashAbsolutePath "$1")
|
|
echo "${abs_path#/viash_automount}"
|
|
}
|
|
# initialise variables
|
|
VIASH_DIRECTORY_MOUNTS=()
|
|
|
|
# initialise docker variables
|
|
VIASH_DOCKER_RUN_ARGS=(-i --rm)
|
|
|
|
# initialise array
|
|
VIASH_POSITIONAL_ARGS=''
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help)
|
|
ViashHelp
|
|
exit
|
|
;;
|
|
---v|---verbose)
|
|
let "VIASH_VERBOSITY=VIASH_VERBOSITY+1"
|
|
shift 1
|
|
;;
|
|
---verbosity)
|
|
VIASH_VERBOSITY="$2"
|
|
shift 2
|
|
;;
|
|
---verbosity=*)
|
|
VIASH_VERBOSITY="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
--version)
|
|
echo "parallel_map main_pipeline"
|
|
exit
|
|
;;
|
|
--input_r1)
|
|
if [ -z "$VIASH_PAR_INPUT_R1" ]; then
|
|
VIASH_PAR_INPUT_R1="$2"
|
|
else
|
|
VIASH_PAR_INPUT_R1="$VIASH_PAR_INPUT_R1;""$2"
|
|
fi
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --input_r1. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--input_r1=*)
|
|
if [ -z "$VIASH_PAR_INPUT_R1" ]; then
|
|
VIASH_PAR_INPUT_R1=$(ViashRemoveFlags "$1")
|
|
else
|
|
VIASH_PAR_INPUT_R1="$VIASH_PAR_INPUT_R1;"$(ViashRemoveFlags "$1")
|
|
fi
|
|
shift 1
|
|
;;
|
|
--input_r2)
|
|
if [ -z "$VIASH_PAR_INPUT_R2" ]; then
|
|
VIASH_PAR_INPUT_R2="$2"
|
|
else
|
|
VIASH_PAR_INPUT_R2="$VIASH_PAR_INPUT_R2;""$2"
|
|
fi
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --input_r2. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--input_r2=*)
|
|
if [ -z "$VIASH_PAR_INPUT_R2" ]; then
|
|
VIASH_PAR_INPUT_R2=$(ViashRemoveFlags "$1")
|
|
else
|
|
VIASH_PAR_INPUT_R2="$VIASH_PAR_INPUT_R2;"$(ViashRemoveFlags "$1")
|
|
fi
|
|
shift 1
|
|
;;
|
|
--genomeDir)
|
|
[ -n "$VIASH_PAR_GENOMEDIR" ] && ViashError Bad arguments for option \'--genomeDir\': \'$VIASH_PAR_GENOMEDIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_GENOMEDIR="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --genomeDir. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--genomeDir=*)
|
|
[ -n "$VIASH_PAR_GENOMEDIR" ] && ViashError Bad arguments for option \'--genomeDir=*\': \'$VIASH_PAR_GENOMEDIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_GENOMEDIR=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--barcodes)
|
|
if [ -z "$VIASH_PAR_BARCODES" ]; then
|
|
VIASH_PAR_BARCODES="$2"
|
|
else
|
|
VIASH_PAR_BARCODES="$VIASH_PAR_BARCODES;""$2"
|
|
fi
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --barcodes. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--barcodes=*)
|
|
if [ -z "$VIASH_PAR_BARCODES" ]; then
|
|
VIASH_PAR_BARCODES=$(ViashRemoveFlags "$1")
|
|
else
|
|
VIASH_PAR_BARCODES="$VIASH_PAR_BARCODES;"$(ViashRemoveFlags "$1")
|
|
fi
|
|
shift 1
|
|
;;
|
|
--wellBarcodesLength)
|
|
[ -n "$VIASH_PAR_WELLBARCODESLENGTH" ] && ViashError Bad arguments for option \'--wellBarcodesLength\': \'$VIASH_PAR_WELLBARCODESLENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_WELLBARCODESLENGTH="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --wellBarcodesLength. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--wellBarcodesLength=*)
|
|
[ -n "$VIASH_PAR_WELLBARCODESLENGTH" ] && ViashError Bad arguments for option \'--wellBarcodesLength=*\': \'$VIASH_PAR_WELLBARCODESLENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_WELLBARCODESLENGTH=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--umiLength)
|
|
[ -n "$VIASH_PAR_UMILENGTH" ] && ViashError Bad arguments for option \'--umiLength\': \'$VIASH_PAR_UMILENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_UMILENGTH="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --umiLength. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--umiLength=*)
|
|
[ -n "$VIASH_PAR_UMILENGTH" ] && ViashError Bad arguments for option \'--umiLength=*\': \'$VIASH_PAR_UMILENGTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_UMILENGTH=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--limitBAMsortRAM)
|
|
[ -n "$VIASH_PAR_LIMITBAMSORTRAM" ] && ViashError Bad arguments for option \'--limitBAMsortRAM\': \'$VIASH_PAR_LIMITBAMSORTRAM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_LIMITBAMSORTRAM="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --limitBAMsortRAM. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--limitBAMsortRAM=*)
|
|
[ -n "$VIASH_PAR_LIMITBAMSORTRAM" ] && ViashError Bad arguments for option \'--limitBAMsortRAM=*\': \'$VIASH_PAR_LIMITBAMSORTRAM\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_LIMITBAMSORTRAM=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--runThreadN)
|
|
[ -n "$VIASH_PAR_RUNTHREADN" ] && ViashError Bad arguments for option \'--runThreadN\': \'$VIASH_PAR_RUNTHREADN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_RUNTHREADN="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --runThreadN. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--runThreadN=*)
|
|
[ -n "$VIASH_PAR_RUNTHREADN" ] && ViashError Bad arguments for option \'--runThreadN=*\': \'$VIASH_PAR_RUNTHREADN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_RUNTHREADN=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--output)
|
|
[ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_OUTPUT="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--output=*)
|
|
[ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--joblog)
|
|
[ -n "$VIASH_PAR_JOBLOG" ] && ViashError Bad arguments for option \'--joblog\': \'$VIASH_PAR_JOBLOG\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_JOBLOG="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --joblog. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--joblog=*)
|
|
[ -n "$VIASH_PAR_JOBLOG" ] && ViashError Bad arguments for option \'--joblog=*\': \'$VIASH_PAR_JOBLOG\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_JOBLOG=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
---engine)
|
|
VIASH_ENGINE_ID="$2"
|
|
shift 2
|
|
;;
|
|
---engine=*)
|
|
VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
---setup)
|
|
VIASH_MODE='setup'
|
|
VIASH_SETUP_STRATEGY="$2"
|
|
shift 2
|
|
;;
|
|
---setup=*)
|
|
VIASH_MODE='setup'
|
|
VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
---dockerfile)
|
|
VIASH_MODE='dockerfile'
|
|
shift 1
|
|
;;
|
|
---debug)
|
|
VIASH_MODE='debug'
|
|
shift 1
|
|
;;
|
|
---cpus)
|
|
[ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_CPUS="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
---cpus=*)
|
|
[ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_CPUS=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
---memory)
|
|
[ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_MEMORY="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
---memory=*)
|
|
[ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_MEMORY=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
*) # positional arg or unknown option
|
|
# since the positional args will be eval'd, can we always quote, instead of using ViashQuote
|
|
VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'"
|
|
[[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters.
|
|
shift # past argument
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# parse positional parameters
|
|
eval set -- $VIASH_POSITIONAL_ARGS
|
|
|
|
|
|
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
|
|
VIASH_ENGINE_TYPE='native'
|
|
elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then
|
|
VIASH_ENGINE_TYPE='docker'
|
|
else
|
|
ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# check if docker is installed properly
|
|
ViashDockerInstallationCheck
|
|
|
|
# determine docker image id
|
|
if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then
|
|
VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/htrnaseq/parallel_map:main_pipeline'
|
|
fi
|
|
|
|
# print dockerfile
|
|
if [ "$VIASH_MODE" == "dockerfile" ]; then
|
|
ViashDockerfile "$VIASH_ENGINE_ID"
|
|
exit 0
|
|
|
|
# enter docker container
|
|
elif [[ "$VIASH_MODE" == "debug" ]]; then
|
|
VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID"
|
|
ViashNotice "+ $VIASH_CMD"
|
|
eval $VIASH_CMD
|
|
exit
|
|
|
|
# build docker image
|
|
elif [ "$VIASH_MODE" == "setup" ]; then
|
|
ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY"
|
|
ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
|
|
exit 0
|
|
fi
|
|
|
|
# check if docker image exists
|
|
ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild
|
|
ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
|
|
fi
|
|
|
|
# setting computational defaults
|
|
|
|
# helper function for parsing memory strings
|
|
function ViashMemoryAsBytes {
|
|
local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'`
|
|
local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$'
|
|
if [[ $memory =~ $memory_regex ]]; then
|
|
local number=${memory/[^0-9]*/}
|
|
local symbol=${memory/*[0-9]/}
|
|
|
|
case $symbol in
|
|
b) memory_b=$number ;;
|
|
kb|k) memory_b=$(( $number * 1000 )) ;;
|
|
mb|m) memory_b=$(( $number * 1000 * 1000 )) ;;
|
|
gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;;
|
|
tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;;
|
|
pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;;
|
|
kib|ki) memory_b=$(( $number * 1024 )) ;;
|
|
mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;;
|
|
gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;;
|
|
tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;;
|
|
pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;;
|
|
esac
|
|
echo "$memory_b"
|
|
fi
|
|
}
|
|
# compute memory in different units
|
|
if [ ! -z ${VIASH_META_MEMORY+x} ]; then
|
|
VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY`
|
|
# do not define other variables if memory_b is an empty string
|
|
if [ ! -z "$VIASH_META_MEMORY_B" ]; then
|
|
VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 ))
|
|
VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 ))
|
|
VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 ))
|
|
VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 ))
|
|
VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 ))
|
|
VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 ))
|
|
VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 ))
|
|
else
|
|
# unset memory if string is empty
|
|
unset $VIASH_META_MEMORY_B
|
|
fi
|
|
fi
|
|
# unset nproc if string is empty
|
|
if [ -z "$VIASH_META_CPUS" ]; then
|
|
unset $VIASH_META_CPUS
|
|
fi
|
|
|
|
|
|
# check whether required parameters exist
|
|
if [ -z ${VIASH_PAR_INPUT_R1+x} ]; then
|
|
ViashError '--input_r1' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_INPUT_R2+x} ]; then
|
|
ViashError '--input_r2' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_GENOMEDIR+x} ]; then
|
|
ViashError '--genomeDir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_BARCODES+x} ]; then
|
|
ViashError '--barcodes' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_WELLBARCODESLENGTH+x} ]; then
|
|
ViashError '--wellBarcodesLength' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_UMILENGTH+x} ]; then
|
|
ViashError '--umiLength' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_OUTPUT+x} ]; then
|
|
ViashError '--output' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_NAME+x} ]; then
|
|
ViashError 'name' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then
|
|
ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then
|
|
ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_EXECUTABLE+x} ]; then
|
|
ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_CONFIG+x} ]; then
|
|
ViashError 'config' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_TEMP_DIR+x} ]; then
|
|
ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
|
|
# filling in defaults
|
|
if [ -z ${VIASH_PAR_LIMITBAMSORTRAM+x} ]; then
|
|
VIASH_PAR_LIMITBAMSORTRAM="10000000000"
|
|
fi
|
|
if [ -z ${VIASH_PAR_RUNTHREADN+x} ]; then
|
|
VIASH_PAR_RUNTHREADN="1"
|
|
fi
|
|
if [ -z ${VIASH_PAR_JOBLOG+x} ]; then
|
|
VIASH_PAR_JOBLOG="execution_log.txt"
|
|
fi
|
|
|
|
# check whether required files exist
|
|
if [ ! -z "$VIASH_PAR_INPUT_R1" ]; then
|
|
IFS=';'
|
|
set -f
|
|
for file in $VIASH_PAR_INPUT_R1; do
|
|
unset IFS
|
|
if [ ! -e "$file" ]; then
|
|
ViashError "Input file '$file' does not exist."
|
|
exit 1
|
|
fi
|
|
done
|
|
set +f
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_INPUT_R2" ]; then
|
|
IFS=';'
|
|
set -f
|
|
for file in $VIASH_PAR_INPUT_R2; do
|
|
unset IFS
|
|
if [ ! -e "$file" ]; then
|
|
ViashError "Input file '$file' does not exist."
|
|
exit 1
|
|
fi
|
|
done
|
|
set +f
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_GENOMEDIR" ] && [ ! -e "$VIASH_PAR_GENOMEDIR" ]; then
|
|
ViashError "Input file '$VIASH_PAR_GENOMEDIR' does not exist."
|
|
exit 1
|
|
fi
|
|
|
|
# check whether parameters values are of the right type
|
|
if [[ -n "$VIASH_PAR_WELLBARCODESLENGTH" ]]; then
|
|
if ! [[ "$VIASH_PAR_WELLBARCODESLENGTH" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError '--wellBarcodesLength' has to be an integer. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_PAR_UMILENGTH" ]]; then
|
|
if ! [[ "$VIASH_PAR_UMILENGTH" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError '--umiLength' has to be an integer. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_PAR_RUNTHREADN" ]]; then
|
|
if ! [[ "$VIASH_PAR_RUNTHREADN" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError '--runThreadN' has to be an integer. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_PAR_OUTPUT" ]]; then
|
|
if ! [[ "$VIASH_PAR_OUTPUT" =~ \* ]]; then
|
|
ViashError '--output' has to be a path containing a wildcard, e.g. 'output_*.txt'. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_CPUS" ]]; then
|
|
if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_B" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_KB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_MB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_GB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_TB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_PB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# create parent directories of output files, if so desired
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then
|
|
mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")"
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_JOBLOG" ] && [ ! -d "$(dirname "$VIASH_PAR_JOBLOG")" ]; then
|
|
mkdir -p "$(dirname "$VIASH_PAR_JOBLOG")"
|
|
fi
|
|
|
|
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
|
|
if [ "$VIASH_MODE" == "run" ]; then
|
|
VIASH_CMD="bash"
|
|
else
|
|
ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# detect volumes from file arguments
|
|
VIASH_CHOWN_VARS=()
|
|
if [ ! -z "$VIASH_PAR_INPUT_R1" ]; then
|
|
VIASH_TEST_INPUT_R1=()
|
|
IFS=';'
|
|
for var in $VIASH_PAR_INPUT_R1; do
|
|
unset IFS
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$var")" )
|
|
var=$(ViashDockerAutodetectMount "$var")
|
|
VIASH_TEST_INPUT_R1+=( "$var" )
|
|
done
|
|
VIASH_PAR_INPUT_R1=$(IFS=';' ; echo "${VIASH_TEST_INPUT_R1[*]}")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_INPUT_R2" ]; then
|
|
VIASH_TEST_INPUT_R2=()
|
|
IFS=';'
|
|
for var in $VIASH_PAR_INPUT_R2; do
|
|
unset IFS
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$var")" )
|
|
var=$(ViashDockerAutodetectMount "$var")
|
|
VIASH_TEST_INPUT_R2+=( "$var" )
|
|
done
|
|
VIASH_PAR_INPUT_R2=$(IFS=';' ; echo "${VIASH_TEST_INPUT_R2[*]}")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_GENOMEDIR" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_GENOMEDIR")" )
|
|
VIASH_PAR_GENOMEDIR=$(ViashDockerAutodetectMount "$VIASH_PAR_GENOMEDIR")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ]; then
|
|
VIASH_TEST_OUTPUT=()
|
|
IFS=';'
|
|
for var in $VIASH_PAR_OUTPUT; do
|
|
unset IFS
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$var")" )
|
|
var=$(ViashDockerAutodetectMount "$var")
|
|
VIASH_TEST_OUTPUT+=( "$var" )
|
|
VIASH_CHOWN_VARS+=( "$var" )
|
|
done
|
|
VIASH_PAR_OUTPUT=$(IFS=';' ; echo "${VIASH_TEST_OUTPUT[*]}")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_JOBLOG" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_JOBLOG")" )
|
|
VIASH_PAR_JOBLOG=$(ViashDockerAutodetectMount "$VIASH_PAR_JOBLOG")
|
|
VIASH_CHOWN_VARS+=( "$VIASH_PAR_JOBLOG" )
|
|
fi
|
|
if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" )
|
|
VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR")
|
|
fi
|
|
if [ ! -z "$VIASH_META_EXECUTABLE" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" )
|
|
VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CONFIG" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" )
|
|
VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG")
|
|
fi
|
|
if [ ! -z "$VIASH_META_TEMP_DIR" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" )
|
|
VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR")
|
|
fi
|
|
|
|
# get unique mounts
|
|
VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u))
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# change file ownership
|
|
function ViashPerformChown {
|
|
if (( ${#VIASH_CHOWN_VARS[@]} )); then
|
|
set +e
|
|
VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'"
|
|
ViashDebug "+ $VIASH_CMD"
|
|
eval $VIASH_CMD
|
|
set -e
|
|
fi
|
|
}
|
|
trap ViashPerformChown EXIT
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# helper function for filling in extra docker args
|
|
if [ ! -z "$VIASH_META_MEMORY_B" ]; then
|
|
VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CPUS" ]; then
|
|
VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}")
|
|
fi
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID"
|
|
fi
|
|
|
|
|
|
# set dependency paths
|
|
|
|
|
|
ViashDebug "Running command: $(echo $VIASH_CMD)"
|
|
cat << VIASHEOF | eval $VIASH_CMD
|
|
set -e
|
|
tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-parallel_map-XXXXXX").sh
|
|
function clean_up {
|
|
rm "\$tempscript"
|
|
}
|
|
function interrupt {
|
|
echo -e "\nCTRL-C Pressed..."
|
|
exit 1
|
|
}
|
|
trap clean_up EXIT
|
|
trap interrupt INT SIGINT
|
|
cat > "\$tempscript" << 'VIASHMAIN'
|
|
#!/bin/bash
|
|
|
|
## VIASH START
|
|
# The following code has been auto-generated by Viash.
|
|
$( if [ ! -z ${VIASH_PAR_INPUT_R1+x} ]; then echo "${VIASH_PAR_INPUT_R1}" | sed "s#'#'\"'\"'#g;s#.*#par_input_r1='&'#" ; else echo "# par_input_r1="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_INPUT_R2+x} ]; then echo "${VIASH_PAR_INPUT_R2}" | sed "s#'#'\"'\"'#g;s#.*#par_input_r2='&'#" ; else echo "# par_input_r2="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_GENOMEDIR+x} ]; then echo "${VIASH_PAR_GENOMEDIR}" | sed "s#'#'\"'\"'#g;s#.*#par_genomeDir='&'#" ; else echo "# par_genomeDir="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_BARCODES+x} ]; then echo "${VIASH_PAR_BARCODES}" | sed "s#'#'\"'\"'#g;s#.*#par_barcodes='&'#" ; else echo "# par_barcodes="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_WELLBARCODESLENGTH+x} ]; then echo "${VIASH_PAR_WELLBARCODESLENGTH}" | sed "s#'#'\"'\"'#g;s#.*#par_wellBarcodesLength='&'#" ; else echo "# par_wellBarcodesLength="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_UMILENGTH+x} ]; then echo "${VIASH_PAR_UMILENGTH}" | sed "s#'#'\"'\"'#g;s#.*#par_umiLength='&'#" ; else echo "# par_umiLength="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_LIMITBAMSORTRAM+x} ]; then echo "${VIASH_PAR_LIMITBAMSORTRAM}" | sed "s#'#'\"'\"'#g;s#.*#par_limitBAMsortRAM='&'#" ; else echo "# par_limitBAMsortRAM="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_RUNTHREADN+x} ]; then echo "${VIASH_PAR_RUNTHREADN}" | sed "s#'#'\"'\"'#g;s#.*#par_runThreadN='&'#" ; else echo "# par_runThreadN="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi )
|
|
$( if [ ! -z ${VIASH_PAR_JOBLOG+x} ]; then echo "${VIASH_PAR_JOBLOG}" | sed "s#'#'\"'\"'#g;s#.*#par_joblog='&'#" ; else echo "# par_joblog="; fi )
|
|
$( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi )
|
|
$( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi )
|
|
$( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "${VIASH_META_RESOURCES_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_resources_dir='&'#" ; else echo "# meta_resources_dir="; fi )
|
|
$( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "${VIASH_META_EXECUTABLE}" | sed "s#'#'\"'\"'#g;s#.*#meta_executable='&'#" ; else echo "# meta_executable="; fi )
|
|
$( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "${VIASH_META_CONFIG}" | sed "s#'#'\"'\"'#g;s#.*#meta_config='&'#" ; else echo "# meta_config="; fi )
|
|
$( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "${VIASH_META_TEMP_DIR}" | sed "s#'#'\"'\"'#g;s#.*#meta_temp_dir='&'#" ; else echo "# meta_temp_dir="; fi )
|
|
$( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "${VIASH_META_CPUS}" | sed "s#'#'\"'\"'#g;s#.*#meta_cpus='&'#" ; else echo "# meta_cpus="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "${VIASH_META_MEMORY_B}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_b='&'#" ; else echo "# meta_memory_b="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "${VIASH_META_MEMORY_KB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kb='&'#" ; else echo "# meta_memory_kb="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "${VIASH_META_MEMORY_MB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mb='&'#" ; else echo "# meta_memory_mb="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "${VIASH_META_MEMORY_GB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gb='&'#" ; else echo "# meta_memory_gb="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "${VIASH_META_MEMORY_TB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tb='&'#" ; else echo "# meta_memory_tb="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "${VIASH_META_MEMORY_PB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pb='&'#" ; else echo "# meta_memory_pb="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "${VIASH_META_MEMORY_KIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_kib='&'#" ; else echo "# meta_memory_kib="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "${VIASH_META_MEMORY_MIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_mib='&'#" ; else echo "# meta_memory_mib="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "${VIASH_META_MEMORY_GIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_gib='&'#" ; else echo "# meta_memory_gib="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_tib='&'#" ; else echo "# meta_memory_tib="; fi )
|
|
$( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi )
|
|
|
|
## VIASH END
|
|
|
|
set -eo pipefail
|
|
|
|
# Check if wildcard character is present in output folder template
|
|
printf "Checking if output folder template (\$par_output) contains a single wildcard character '*'. "
|
|
output_glob_character="\${par_output//[^\\*]}"
|
|
if [[ "\${#output_glob_character}" -ne "1" ]]; then
|
|
echo "The value for --output must contain exactly one '*' character. Exiting..."
|
|
exit 1
|
|
else
|
|
echo "Done, wildcard character found!"
|
|
fi
|
|
|
|
# Split the delimited strings into arrays
|
|
IFS=';' read -r -a barcodes <<< "\$par_barcodes"
|
|
IFS=';' read -r -a input_r1 <<< "\$par_input_r1"
|
|
IFS=';' read -r -a input_r2 <<< "\$par_input_r2"
|
|
|
|
# Check that the number of values provided for the barcodes and the fastq files are the same.
|
|
num_barcodes="\${#barcodes[@]}"
|
|
num_r1_inputs="\${#input_r1[@]}"
|
|
num_r2_inputs="\${#input_r2[@]}"
|
|
|
|
if [ ! "\$num_barcodes" -eq "\$num_r1_inputs" ] || [ ! "\$num_r1_inputs" -eq "\$num_r2_inputs" ]; then
|
|
echo "The number of values for arguments 'barcodes' (\$num_barcodes), "\\
|
|
"'input_r1' (\$num_r1_inputs) and 'input_r2' (\$num_r2_inputs) "\\
|
|
"should be the same, and their order should match."
|
|
exit 1
|
|
else
|
|
echo "Checked if length of barcodes input (\$num_barcodes) is "\\
|
|
"the same as R1 reads (\$num_r1_inputs) and R2 reads "\\
|
|
"(\$num_r2_inputs). Seems OK!"
|
|
fi
|
|
|
|
# Function to test for unique values in array
|
|
function arrayContainsUniqueValues {
|
|
# Pass the argument by reference
|
|
local -n arr=\$1
|
|
# Create a temporary associative array
|
|
# in order to use its uniqueness of keys
|
|
# 'declare' in a function is automatically local
|
|
declare -A uniq_tmp
|
|
for item in "\${arr[@]}"; do
|
|
uniq_tmp[\$item]=0 # assigning a placeholder
|
|
done
|
|
local unique_array_values=(\${!uniq_tmp[@]})
|
|
if [ "\${#unique_array_values[@]}" -eq "\${#arr[@]}" ]; then
|
|
return
|
|
fi
|
|
false
|
|
}
|
|
arrayContainsUniqueValues barcodes
|
|
is_array_unique_exit_code=\$?
|
|
if ! (exit \$is_array_unique_exit_code); then
|
|
echo "The provided barcodes should be unique!"
|
|
echo "Values: \$par_barcodes"
|
|
exit 1
|
|
fi
|
|
|
|
# Define the function that will be used to run a single job
|
|
function _run() {
|
|
local par_wellBarcodeLength="\$1"
|
|
local par_UMIlength="\$2"
|
|
local par_output="\$3"
|
|
local par_genomeDir="\$4"
|
|
local par_limitBAMsortRAM="\$5"
|
|
local par_runThreadN="\$6"
|
|
local barcode="\$7"
|
|
local input_R1="\$8"
|
|
local input_R2="\$9"
|
|
local par_UMIstart=\$((\$par_wellBarcodeLength + 1))
|
|
|
|
set -eo pipefail
|
|
|
|
echo <<-EOF
|
|
Processing \$barcode
|
|
For the following inputs (lanes):
|
|
"\$star_readFilesIn
|
|
EOF
|
|
|
|
echo "Writing barcode '\$barcode' to \$barcode.txt and using it as input".
|
|
# Note that there is no possible conflict between jobs here
|
|
# because the barcodes are unique (and the barcode is part of the name
|
|
# of the file).
|
|
echo "\$barcode" > "\$barcode.txt"
|
|
|
|
local dir="\${par_output//\\*/\$barcode}/"
|
|
echo "Setting output for barcode '\$barcode' to '\$dir'."
|
|
mkdir -p "\$dir"
|
|
|
|
# check if files are compressed
|
|
local TMPDIR=\$(mktemp -d "\$meta_temp_dir/parallel_map-\$barcode-XXXXXX")
|
|
function clean_up {
|
|
[[ -d "\$TMPDIR" ]] && rm -r "\$TMPDIR"
|
|
}
|
|
trap clean_up RETURN
|
|
|
|
# Decompress the input files when needed
|
|
# NOTE: for some reason, using STAR's --readFilesCommand does not always work
|
|
# This might be because STAR creates fifo files (see https://man7.org/linux/man-pages/man7/fifo.7.html)
|
|
# and this requires a filesystem that supports this. Another cause might be that the input files
|
|
# are symlinks. When testing this, using '--readFilesCommand "zcat"'
|
|
# always produced empty BAM files, but also a succesfull exit code (0) so the problem is not reported.
|
|
# However, the logs showed the following error: "gzip -: unexpected end of file".
|
|
|
|
function is_gzipped {
|
|
printf "Checking if input '\$1' (barcode '\$barcode') is gzipped... "
|
|
if file "\$1" | grep -q 'gzip'; then
|
|
echo "Done, detected compressed file."
|
|
return
|
|
fi
|
|
echo "Done, file does not need decompression."
|
|
false
|
|
}
|
|
|
|
# Resolve symbolic links to actual file paths
|
|
input_R1=\$(realpath \$input_R1)
|
|
input_R2=\$(realpath \$input_R2)
|
|
|
|
if is_gzipped \$input_R1; then
|
|
local compressed_file_name_r1="\$(basename -- \$input_R1)"
|
|
local uncompressed_file_r1="\$TMPDIR/\${compressed_file_name_r1%.gz}"
|
|
printf "Unpacking input to \$uncompressed_file_r1... "
|
|
zcat "\$input_R1" > "\$uncompressed_file_r1"
|
|
echo "Decompression done."
|
|
else
|
|
local uncompressed_file_r1="\$input_R1"
|
|
fi
|
|
|
|
if is_gzipped \$input_R2; then
|
|
local compressed_file_name_r2="\$(basename -- \$input_R2)"
|
|
local uncompressed_file_r2="\$TMPDIR/\${compressed_file_name_r2%.gz}"
|
|
printf "Unpacking input to \$uncompressed_file_r2... "
|
|
zcat "\$input_R2" > "\$uncompressed_file_r2"
|
|
echo "Decompression done."
|
|
else
|
|
local uncompressed_file_r2="\$input_R2"
|
|
fi
|
|
|
|
local n_input_lines_r1=\$(wc -l < "\$uncompressed_file_r1")
|
|
local n_input_lines_r2=\$(wc -l < "\$uncompressed_file_r2")
|
|
|
|
printf "Checking if length of input file mates match. "
|
|
if (( \$n_input_lines_r1 != n_input_lines_r2 )); then
|
|
echo "The length of file \$input_R1 (\$n_input_lines_r1) does not match with \$input_R2 (\$n_input_lines_r2)"
|
|
return 1
|
|
else
|
|
echo "Seems OK, \$n_input_lines_r1 input lines."
|
|
fi
|
|
echo "Starting STAR for barcode '\$barcode'"
|
|
# soloType 'Droplet' is the same as 'CB_UMI_Simple': one UMI and one cell barcode of fixed length.
|
|
# By default in this mode, STAR will look for the cell barcode and the UMI int the last files specified with --readFilesIn
|
|
# So we need to specify R2 first and R1 second, because R1 contains the barcode and UMI.
|
|
# Also, you might be tempted to use '--soloBarcodeMate 1' to alter this behavior, but this requires the clipping
|
|
# the barcode from this mate by specifying --clip5pNbases and/or --clip3pNbases, which we do not want to do.
|
|
STAR \\
|
|
--readFilesIn "\$uncompressed_file_r2" "\$uncompressed_file_r1" \\
|
|
--soloType Droplet \\
|
|
--quantMode GeneCounts \\
|
|
--genomeLoad LoadAndKeep \\
|
|
--limitBAMsortRAM "\$par_limitBAMsortRAM" \\
|
|
--runThreadN "\$par_runThreadN" \\
|
|
--outFilterMultimapNmax 1 \\
|
|
--outSAMtype BAM SortedByCoordinate \\
|
|
--soloCBstart 1 \\
|
|
--readFilesType "Fastx" \\
|
|
--soloCBlen "\$par_wellBarcodeLength" \\
|
|
--soloUMIstart "\$par_UMIstart" \\
|
|
--soloUMIlen "\$par_UMIlength" \\
|
|
--soloBarcodeReadLength 0 \\
|
|
--soloStrand Unstranded \\
|
|
--soloFeatures Gene \\
|
|
--genomeDir "\$par_genomeDir" \\
|
|
--outReadsUnmapped Fastx \\
|
|
--outSAMunmapped Within \\
|
|
--outSAMattributes NH HI nM AS CR UR CB UB GX GN \\
|
|
--soloCBwhitelist "\$barcode.txt" \\
|
|
--outFileNamePrefix "\$dir" \\
|
|
--outTmpDir "\$TMPDIR/STARtemp/"
|
|
|
|
printf "Done running STAR. "
|
|
# Check if the number of processed reads is equal to the number of input reads
|
|
local n_input_reads=\$((\$n_input_lines_r1 / 4))
|
|
local nr_output_reads=\$(grep -Po "Number\\ of\\ input\\ reads \\\\|\\W*\\K\\d+" "\$dir/Log.final.out")
|
|
if (( \$nr_output_reads != \$n_input_reads )); then
|
|
echo "Not all input reads were processed for barcode \$barcode."
|
|
return 1
|
|
else
|
|
echo "Processed \$nr_output_reads reads for barcode \$barcode".
|
|
fi
|
|
}
|
|
|
|
# Export the function - requires bash
|
|
export -f _run
|
|
|
|
# Load reference genome
|
|
echo "Loading reference genome"
|
|
STAR --genomeLoad LoadAndExit --genomeDir "\$par_genomeDir"
|
|
|
|
# Run the concurrent jobs using GNU parallel
|
|
|
|
# Make sure that parallel uses the correct shell
|
|
export PARALLEL_SHELL="/bin/bash"
|
|
|
|
# Some notes:
|
|
# --halt now,fail=1: instruct parallel to exit when a job has failed and kill remaining running jobs.
|
|
#
|
|
# ::: is a special syntax for GNU parallel to delineate inputs
|
|
# If multiple ::: are given, each group will be treated as an input source, and all combinations of input
|
|
# sources will be generated. E.g. ::: 1 2 ::: a b c will result in the combinations (1,a) (1,b) (1,c) (2,a) (2,b) (2,c)
|
|
# The delimiter :::+ (note the extra '+') links the argument to the previous argument, and one argument from each of the input
|
|
# sources will be read.
|
|
parallel_cmd=("parallel" "--jobs" "80%" "--verbose" "--memfree" "2G"
|
|
"--tmpdir" "\$meta_temp_dir"
|
|
"--retry-failed" "--retries" "4" "--halt" "soon,fail=1"
|
|
"--joblog" "\$par_joblog" "_run" "{}")
|
|
|
|
# Arguments for which there is one value, so these will not create extra jobs
|
|
parallel_cmd+=(":::" "\$par_wellBarcodesLength" ":::" "\$par_umiLength" ":::" "\$par_output" ":::" "\$par_genomeDir" ":::" "\$par_limitBAMsortRAM" ":::" "\$par_runThreadN")
|
|
|
|
# Argument which in fact will cause extra jobs to be spawned, per job one item from each argument will be selected
|
|
# Thus, these argument lists should have the same length.
|
|
parallel_cmd+=(":::" "\${barcodes[@]}" ":::+" "\${input_r1[@]}" ":::+" "\${input_r2[@]}")
|
|
|
|
set +eo pipefail
|
|
"\${parallel_cmd[@]}"
|
|
exit_code=\$?
|
|
set -eo pipefail
|
|
|
|
echo "GNU parallel finished!"
|
|
|
|
# Unload reference
|
|
printf "Unloading reference genome. "
|
|
STAR --genomeLoad Remove --genomeDir "\$par_genomeDir"
|
|
echo "Done!"
|
|
|
|
# Exit code from GNU parallel:
|
|
# If fail=1 is used, the exit status will be the exit status of the failing job.
|
|
echo "Checking exit code"
|
|
if ((exit_code>0)); then
|
|
# Note that the ending HERE must be indented with TAB characters (not spaces)
|
|
# in order to remove leading indentation
|
|
MESSAGE=\$(
|
|
cat <<-HERE
|
|
==================================================================
|
|
|
|
!!! An error occurred for one of the jobs.
|
|
Exit code of the failing job: \$exit_code.
|
|
|
|
%s
|
|
|
|
==================================================================
|
|
|
|
HERE
|
|
)
|
|
printf "\$MESSAGE" "\$(<\$par_joblog)"
|
|
exit 1
|
|
else
|
|
cat <<-HERE
|
|
==================================================================
|
|
|
|
Mapping went fine (exit code '\$exit_code'), zero errors occurred
|
|
|
|
==================================================================
|
|
HERE
|
|
|
|
fi
|
|
VIASHMAIN
|
|
bash "\$tempscript" &
|
|
wait "\$!"
|
|
|
|
VIASHEOF
|
|
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# strip viash automount from file paths
|
|
|
|
if [ ! -z "$VIASH_PAR_INPUT_R1" ]; then
|
|
unset VIASH_TEST_INPUT_R1
|
|
IFS=';'
|
|
for var in $VIASH_PAR_INPUT_R1; do
|
|
unset IFS
|
|
if [ -z "$VIASH_TEST_INPUT_R1" ]; then
|
|
VIASH_TEST_INPUT_R1="$(ViashDockerStripAutomount "$var")"
|
|
else
|
|
VIASH_TEST_INPUT_R1="$VIASH_TEST_INPUT_R1;""$(ViashDockerStripAutomount "$var")"
|
|
fi
|
|
done
|
|
VIASH_PAR_INPUT_R1="$VIASH_TEST_INPUT_R1"
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_INPUT_R2" ]; then
|
|
unset VIASH_TEST_INPUT_R2
|
|
IFS=';'
|
|
for var in $VIASH_PAR_INPUT_R2; do
|
|
unset IFS
|
|
if [ -z "$VIASH_TEST_INPUT_R2" ]; then
|
|
VIASH_TEST_INPUT_R2="$(ViashDockerStripAutomount "$var")"
|
|
else
|
|
VIASH_TEST_INPUT_R2="$VIASH_TEST_INPUT_R2;""$(ViashDockerStripAutomount "$var")"
|
|
fi
|
|
done
|
|
VIASH_PAR_INPUT_R2="$VIASH_TEST_INPUT_R2"
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_GENOMEDIR" ]; then
|
|
VIASH_PAR_GENOMEDIR=$(ViashDockerStripAutomount "$VIASH_PAR_GENOMEDIR")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ]; then
|
|
VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_JOBLOG" ]; then
|
|
VIASH_PAR_JOBLOG=$(ViashDockerStripAutomount "$VIASH_PAR_JOBLOG")
|
|
fi
|
|
if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then
|
|
VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR")
|
|
fi
|
|
if [ ! -z "$VIASH_META_EXECUTABLE" ]; then
|
|
VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CONFIG" ]; then
|
|
VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG")
|
|
fi
|
|
if [ ! -z "$VIASH_META_TEMP_DIR" ]; then
|
|
VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR")
|
|
fi
|
|
fi
|
|
|
|
|
|
# check whether required files exist
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ] && ! compgen -G "$VIASH_PAR_OUTPUT" > /dev/null; then
|
|
ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist."
|
|
exit 1
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_JOBLOG" ] && [ ! -e "$VIASH_PAR_JOBLOG" ]; then
|
|
ViashError "Output file '$VIASH_PAR_JOBLOG' does not exist."
|
|
exit 1
|
|
fi
|
|
|
|
|
|
exit 0
|