CI
44dc2ea48d
Build pipeline: viash-hub.htrnaseq.v0.14.6-q286w
Source commit: 9346c55e3f
Source message: Bump version to v0.14.6
1657 lines
62 KiB
Bash
Executable File
1657 lines
62 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# create_eset v0.14.6
|
|
#
|
|
# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative
|
|
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
|
|
# Intuitive.
|
|
#
|
|
# The component may contain files which fall under a different license. The
|
|
# authors of this component should specify the license in the header of such
|
|
# files, or include a separate license file detailing the licenses of all included
|
|
# files.
|
|
#
|
|
# Component authors:
|
|
# * Dries Schaumont (maintainer)
|
|
# * Marijke Van Moerbeke (author)
|
|
|
|
set -e
|
|
|
|
if [ -z "$VIASH_TEMP" ]; then
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR}
|
|
VIASH_TEMP=${VIASH_TEMP:-$TEMP}
|
|
VIASH_TEMP=${VIASH_TEMP:-/tmp}
|
|
fi
|
|
|
|
# define helper functions
|
|
# ViashQuote: put quotes around non flag values
|
|
# $1 : unquoted string
|
|
# return : possibly quoted string
|
|
# examples:
|
|
# ViashQuote --foo # returns --foo
|
|
# ViashQuote bar # returns 'bar'
|
|
# Viashquote --foo=bar # returns --foo='bar'
|
|
function ViashQuote {
|
|
if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then
|
|
echo "$1" | sed "s#=\(.*\)#='\1'#"
|
|
elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then
|
|
echo "$1"
|
|
else
|
|
echo "'$1'"
|
|
fi
|
|
}
|
|
# ViashRemoveFlags: Remove leading flag
|
|
# $1 : string with a possible leading flag
|
|
# return : string without possible leading flag
|
|
# examples:
|
|
# ViashRemoveFlags --foo=bar # returns bar
|
|
function ViashRemoveFlags {
|
|
echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//'
|
|
}
|
|
# ViashSourceDir: return the path of a bash file, following symlinks
|
|
# usage : ViashSourceDir ${BASH_SOURCE[0]}
|
|
# $1 : Should always be set to ${BASH_SOURCE[0]}
|
|
# returns : The absolute path of the bash file
|
|
function ViashSourceDir {
|
|
local source="$1"
|
|
while [ -h "$source" ]; do
|
|
local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )"
|
|
source="$(readlink "$source")"
|
|
[[ $source != /* ]] && source="$dir/$source"
|
|
done
|
|
cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd
|
|
}
|
|
# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks
|
|
# usage : ViashFindTargetDir 'ScriptPath'
|
|
# $1 : The location from where to start the upward search
|
|
# returns : The absolute path of the '.build.yaml' file
|
|
function ViashFindTargetDir {
|
|
local source="$1"
|
|
while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do
|
|
source=${source%/*}
|
|
done
|
|
echo $source
|
|
}
|
|
# see https://en.wikipedia.org/wiki/Syslog#Severity_level
|
|
VIASH_LOGCODE_EMERGENCY=0
|
|
VIASH_LOGCODE_ALERT=1
|
|
VIASH_LOGCODE_CRITICAL=2
|
|
VIASH_LOGCODE_ERROR=3
|
|
VIASH_LOGCODE_WARNING=4
|
|
VIASH_LOGCODE_NOTICE=5
|
|
VIASH_LOGCODE_INFO=6
|
|
VIASH_LOGCODE_DEBUG=7
|
|
VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE
|
|
|
|
# ViashLog: Log events depending on the verbosity level
|
|
# usage: ViashLog 1 alert Oh no something went wrong!
|
|
# $1: required verbosity level
|
|
# $2: display tag
|
|
# $3+: messages to display
|
|
# stdout: Your input, prepended by '[$2] '.
|
|
function ViashLog {
|
|
local required_level="$1"
|
|
local display_tag="$2"
|
|
shift 2
|
|
if [ $VIASH_VERBOSITY -ge $required_level ]; then
|
|
>&2 echo "[$display_tag]" "$@"
|
|
fi
|
|
}
|
|
|
|
# ViashEmergency: log events when the system is unstable
|
|
# usage: ViashEmergency Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[emergency] '.
|
|
function ViashEmergency {
|
|
ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@"
|
|
}
|
|
|
|
# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database)
|
|
# usage: ViashAlert Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[alert] '.
|
|
function ViashAlert {
|
|
ViashLog $VIASH_LOGCODE_ALERT alert "$@"
|
|
}
|
|
|
|
# ViashCritical: log events when a critical condition occurs
|
|
# usage: ViashCritical Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[critical] '.
|
|
function ViashCritical {
|
|
ViashLog $VIASH_LOGCODE_CRITICAL critical "$@"
|
|
}
|
|
|
|
# ViashError: log events when an error condition occurs
|
|
# usage: ViashError Oh no something went wrong.
|
|
# stdout: Your input, prepended by '[error] '.
|
|
function ViashError {
|
|
ViashLog $VIASH_LOGCODE_ERROR error "$@"
|
|
}
|
|
|
|
# ViashWarning: log potentially abnormal events
|
|
# usage: ViashWarning Something may have gone wrong.
|
|
# stdout: Your input, prepended by '[warning] '.
|
|
function ViashWarning {
|
|
ViashLog $VIASH_LOGCODE_WARNING warning "$@"
|
|
}
|
|
|
|
# ViashNotice: log significant but normal events
|
|
# usage: ViashNotice This just happened.
|
|
# stdout: Your input, prepended by '[notice] '.
|
|
function ViashNotice {
|
|
ViashLog $VIASH_LOGCODE_NOTICE notice "$@"
|
|
}
|
|
|
|
# ViashInfo: log normal events
|
|
# usage: ViashInfo This just happened.
|
|
# stdout: Your input, prepended by '[info] '.
|
|
function ViashInfo {
|
|
ViashLog $VIASH_LOGCODE_INFO info "$@"
|
|
}
|
|
|
|
# ViashDebug: log all events, for debugging purposes
|
|
# usage: ViashDebug This just happened.
|
|
# stdout: Your input, prepended by '[debug] '.
|
|
function ViashDebug {
|
|
ViashLog $VIASH_LOGCODE_DEBUG debug "$@"
|
|
}
|
|
|
|
# find source folder of this component
|
|
VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}`
|
|
|
|
# find the root of the built components & dependencies
|
|
VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR`
|
|
|
|
# define meta fields
|
|
VIASH_META_NAME="create_eset"
|
|
VIASH_META_FUNCTIONALITY_NAME="create_eset"
|
|
VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME"
|
|
VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml"
|
|
VIASH_META_TEMP_DIR="$VIASH_TEMP"
|
|
|
|
|
|
|
|
# initialise variables
|
|
VIASH_MODE='run'
|
|
VIASH_ENGINE_ID='docker'
|
|
|
|
######## Helper functions for setting up Docker images for viash ########
|
|
# expects: ViashDockerBuild
|
|
|
|
# ViashDockerInstallationCheck: check whether Docker is installed correctly
|
|
#
|
|
# examples:
|
|
# ViashDockerInstallationCheck
|
|
function ViashDockerInstallationCheck {
|
|
ViashDebug "Checking whether Docker is installed"
|
|
if [ ! command -v docker &> /dev/null ]; then
|
|
ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions."
|
|
exit 1
|
|
fi
|
|
|
|
ViashDebug "Checking whether the Docker daemon is running"
|
|
local save=$-; set +e
|
|
local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null)
|
|
local out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashCritical "Docker daemon does not seem to be running. Try one of the following:"
|
|
ViashCritical "- Try running 'dockerd' in the command line"
|
|
ViashCritical "- See https://docs.docker.com/config/daemon/"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerRemoteTagCheck: check whether a Docker image is available
|
|
# on a remote. Assumes `docker login` has been performed, if relevant.
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerRemoteTagCheck python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerRemoteTagCheck sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerRemoteTagCheck {
|
|
docker manifest inspect $1 > /dev/null 2> /dev/null
|
|
}
|
|
|
|
# ViashDockerLocalTagCheck: check whether a Docker image is available locally
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# docker pull python:latest
|
|
# ViashDockerLocalTagCheck python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerLocalTagCheck sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerLocalTagCheck {
|
|
[ -n "$(docker images -q $1)" ]
|
|
}
|
|
|
|
# ViashDockerPull: pull a Docker image
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerPull python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerPull sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerPull {
|
|
ViashNotice "Checking if Docker image is available at '$1'"
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
docker pull $1 && return 0 || return 1
|
|
else
|
|
local save=$-; set +e
|
|
docker pull $1 2> /dev/null > /dev/null
|
|
local out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible."
|
|
fi
|
|
return $out
|
|
fi
|
|
}
|
|
|
|
# ViashDockerPush: push a Docker image
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# exit code $? : whether or not the image was found
|
|
# examples:
|
|
# ViashDockerPush python:latest
|
|
# echo $? # returns '0'
|
|
# ViashDockerPush sdaizudceahifu
|
|
# echo $? # returns '1'
|
|
function ViashDockerPush {
|
|
ViashNotice "Pushing image to '$1'"
|
|
local save=$-; set +e
|
|
local out
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
docker push $1
|
|
out=$?
|
|
else
|
|
docker push $1 2> /dev/null > /dev/null
|
|
out=$?
|
|
fi
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -eq 0 ]; then
|
|
ViashNotice "Container '$1' push succeeded."
|
|
else
|
|
ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions."
|
|
fi
|
|
return $out
|
|
}
|
|
|
|
# ViashDockerPullElseBuild: pull a Docker image, else build it
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument.
|
|
# examples:
|
|
# ViashDockerPullElseBuild mynewcomponent
|
|
function ViashDockerPullElseBuild {
|
|
local save=$-; set +e
|
|
ViashDockerPull $1
|
|
local out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashDockerBuild $@
|
|
fi
|
|
}
|
|
|
|
# ViashDockerSetup: create a Docker image, according to specified docker setup strategy
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $2 : docker setup strategy, see DockerSetupStrategy.scala
|
|
# examples:
|
|
# ViashDockerSetup mynewcomponent alwaysbuild
|
|
function ViashDockerSetup {
|
|
local image_id="$1"
|
|
local setup_strategy="$2"
|
|
if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then
|
|
ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then
|
|
ViashDockerPull $image_id
|
|
elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then
|
|
ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then
|
|
ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then
|
|
ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then
|
|
local save=$-; set +e
|
|
ViashDockerLocalTagCheck $image_id
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -eq 0 ]; then
|
|
ViashInfo "Image $image_id already exists"
|
|
elif [ "$setup_strategy" == "ifneedbebuild" ]; then
|
|
ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then
|
|
ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbepull" ]; then
|
|
ViashDockerPull $image_id
|
|
elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then
|
|
ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id")
|
|
elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then
|
|
ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id")
|
|
else
|
|
ViashError "Unrecognised Docker strategy: $setup_strategy"
|
|
exit 1
|
|
fi
|
|
elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then
|
|
ViashDockerPush "$image_id"
|
|
elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then
|
|
local save=$-; set +e
|
|
ViashDockerRemoteTagCheck $image_id
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -eq 0 ]; then
|
|
ViashNotice "Container '$image_id' exists, doing nothing."
|
|
else
|
|
ViashNotice "Container '$image_id' does not yet exist."
|
|
ViashDockerPush "$image_id"
|
|
fi
|
|
elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then
|
|
ViashNotice "Skipping setup."
|
|
else
|
|
ViashError "Unrecognised Docker strategy: $setup_strategy"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerCheckCommands: Check whether a docker container has the required commands
|
|
#
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $@ : commands to verify being present
|
|
# examples:
|
|
# ViashDockerCheckCommands bash:4.0 bash ps foo
|
|
function ViashDockerCheckCommands {
|
|
local image_id="$1"
|
|
shift 1
|
|
local commands="$@"
|
|
local save=$-; set +e
|
|
local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0'
|
|
missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done")
|
|
local outCheck=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $outCheck -ne 0 ]; then
|
|
ViashError "Docker container '$image_id' does not contain command '$missing'."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# ViashDockerBuild: build a docker image
|
|
# $1 : image identifier with format `[registry/]image[:tag]`
|
|
# $... : additional arguments to pass to docker build
|
|
# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in
|
|
# $VIASH_META_NAME : name of the component
|
|
# $VIASH_META_RESOURCES_DIR : directory containing the resources
|
|
# $VIASH_VERBOSITY : verbosity level
|
|
# exit code $? : whether or not the image was built successfully
|
|
function ViashDockerBuild {
|
|
local image_id="$1"
|
|
shift 1
|
|
|
|
# create temporary directory to store dockerfile & optional resources in
|
|
local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX")
|
|
local dockerfile="$tmpdir/Dockerfile"
|
|
function clean_up {
|
|
rm -rf "$tmpdir"
|
|
}
|
|
trap clean_up EXIT
|
|
|
|
# store dockerfile and resources
|
|
ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile"
|
|
|
|
# generate the build command
|
|
local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'"
|
|
|
|
# build the container
|
|
ViashNotice "Building container '$image_id' with Dockerfile"
|
|
ViashInfo "$docker_build_cmd"
|
|
local save=$-; set +e
|
|
if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then
|
|
eval $docker_build_cmd
|
|
else
|
|
eval $docker_build_cmd &> "$tmpdir/docker_build.log"
|
|
fi
|
|
|
|
# check exit code
|
|
local out=$?
|
|
[[ $save =~ e ]] && set -e
|
|
if [ $out -ne 0 ]; then
|
|
ViashError "Error occurred while building container '$image_id'"
|
|
if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then
|
|
ViashError "Transcript: --------------------------------"
|
|
cat "$tmpdir/docker_build.log"
|
|
ViashError "End of transcript --------------------------"
|
|
fi
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
######## End of helper functions for setting up Docker images for viash ########
|
|
|
|
# ViashDockerFile: print the dockerfile to stdout
|
|
# $1 : engine identifier
|
|
# return : dockerfile required to run this component
|
|
# examples:
|
|
# ViashDockerFile
|
|
function ViashDockerfile {
|
|
local engine_id="$1"
|
|
|
|
if [[ "$engine_id" == "docker" ]]; then
|
|
cat << 'VIASHDOCKER'
|
|
FROM rocker/r2u:24.04
|
|
ENTRYPOINT []
|
|
RUN Rscript -e 'options(warn = 2); if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")' && \
|
|
Rscript -e 'options(warn = 2); if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")' && \
|
|
Rscript -e 'options(warn = 2); if (!requireNamespace("Seurat", quietly = TRUE)) BiocManager::install("Seurat")' && \
|
|
Rscript -e 'options(warn = 2); remotes::install_cran(c("data.table", "nlcv"), repos = "https://cran.rstudio.com")'
|
|
|
|
LABEL org.opencontainers.image.authors="Dries Schaumont, Marijke Van Moerbeke"
|
|
LABEL org.opencontainers.image.description="Companion container for running component eset create_eset"
|
|
LABEL org.opencontainers.image.created="2026-02-23T13:37:11Z"
|
|
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
|
|
LABEL org.opencontainers.image.revision="9346c55e3f894994935b0928759dca9e56866d37"
|
|
LABEL org.opencontainers.image.version="v0.14.6"
|
|
|
|
VIASHDOCKER
|
|
fi
|
|
}
|
|
|
|
# ViashDockerBuildArgs: return the arguments to pass to docker build
|
|
# $1 : engine identifier
|
|
# return : arguments to pass to docker build
|
|
function ViashDockerBuildArgs {
|
|
local engine_id="$1"
|
|
|
|
if [[ "$engine_id" == "docker" ]]; then
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# ViashAbsolutePath: generate absolute path from relative path
|
|
# borrowed from https://stackoverflow.com/a/21951256
|
|
# $1 : relative filename
|
|
# return : absolute path
|
|
# examples:
|
|
# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt
|
|
# ViashAbsolutePath /foo/bar/.. # returns /foo
|
|
function ViashAbsolutePath {
|
|
local thePath
|
|
local parr
|
|
local outp
|
|
local len
|
|
if [[ ! "$1" =~ ^/ ]]; then
|
|
thePath="$PWD/$1"
|
|
else
|
|
thePath="$1"
|
|
fi
|
|
echo "$thePath" | (
|
|
IFS=/
|
|
read -a parr
|
|
declare -a outp
|
|
for i in "${parr[@]}"; do
|
|
case "$i" in
|
|
''|.) continue ;;
|
|
..)
|
|
len=${#outp[@]}
|
|
if ((len==0)); then
|
|
continue
|
|
else
|
|
unset outp[$((len-1))]
|
|
fi
|
|
;;
|
|
*)
|
|
len=${#outp[@]}
|
|
outp[$len]="$i"
|
|
;;
|
|
esac
|
|
done
|
|
echo /"${outp[*]}"
|
|
)
|
|
}
|
|
# ViashDockerAutodetectMount: auto configuring docker mounts from parameters
|
|
# $1 : The parameter value
|
|
# returns : New parameter
|
|
# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker
|
|
# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts
|
|
# examples:
|
|
# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar'
|
|
# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"'
|
|
function ViashDockerAutodetectMount {
|
|
local abs_path=$(ViashAbsolutePath "$1")
|
|
local mount_source
|
|
local base_name
|
|
if [ -d "$abs_path" ]; then
|
|
mount_source="$abs_path"
|
|
base_name=""
|
|
else
|
|
mount_source=`dirname "$abs_path"`
|
|
base_name=`basename "$abs_path"`
|
|
fi
|
|
local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source"
|
|
if [ -z "$base_name" ]; then
|
|
echo "$mount_target"
|
|
else
|
|
echo "$mount_target/$base_name"
|
|
fi
|
|
}
|
|
function ViashDockerAutodetectMountArg {
|
|
local abs_path=$(ViashAbsolutePath "$1")
|
|
local mount_source
|
|
local base_name
|
|
if [ -d "$abs_path" ]; then
|
|
mount_source="$abs_path"
|
|
base_name=""
|
|
else
|
|
mount_source=`dirname "$abs_path"`
|
|
base_name=`basename "$abs_path"`
|
|
fi
|
|
local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source"
|
|
ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target"
|
|
echo "--volume=\"$mount_source:$mount_target\""
|
|
}
|
|
function ViashDockerStripAutomount {
|
|
local abs_path=$(ViashAbsolutePath "$1")
|
|
echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}"
|
|
}
|
|
# initialise variables
|
|
VIASH_DIRECTORY_MOUNTS=()
|
|
|
|
# configure default docker automount prefix if it is unset
|
|
if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then
|
|
VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount"
|
|
fi
|
|
|
|
# initialise docker variables
|
|
VIASH_DOCKER_RUN_ARGS=(-i --rm)
|
|
|
|
|
|
# ViashHelp: Display helpful explanation about this executable
|
|
function ViashHelp {
|
|
echo "create_eset v0.14.6"
|
|
echo ""
|
|
echo "Arguments:"
|
|
echo " --pDataFile"
|
|
echo " type: file, required parameter, file must exist"
|
|
echo ""
|
|
echo " --fDataFile"
|
|
echo " type: file, required parameter, file must exist"
|
|
echo ""
|
|
echo " --mappingDir"
|
|
echo " type: file, required parameter, multiple values allowed, file must exist"
|
|
echo ""
|
|
echo " --poolName"
|
|
echo " type: string, required parameter"
|
|
echo ""
|
|
echo " --output"
|
|
echo " type: file, required parameter, output, file must exist"
|
|
echo " default: eset.\$id.rds"
|
|
echo ""
|
|
echo "Viash built in Computational Requirements:"
|
|
echo " ---cpus=INT"
|
|
echo " Number of CPUs to use"
|
|
echo " ---memory=STRING"
|
|
echo " Amount of memory to use. Examples: 4GB, 3MiB."
|
|
echo ""
|
|
echo "Viash built in Docker:"
|
|
echo " ---setup=STRATEGY"
|
|
echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing."
|
|
echo " Default: ifneedbepullelsecachedbuild"
|
|
echo " ---dockerfile"
|
|
echo " Print the dockerfile to stdout."
|
|
echo " ---docker_run_args=ARG"
|
|
echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information."
|
|
echo " ---docker_image_id"
|
|
echo " Print the docker image id to stdout."
|
|
echo " ---debug"
|
|
echo " Enter the docker container for debugging purposes."
|
|
echo ""
|
|
echo "Viash built in Engines:"
|
|
echo " ---engine=ENGINE_ID"
|
|
echo " Specify the engine to use. Options are: docker, native."
|
|
echo " Default: docker"
|
|
}
|
|
|
|
# initialise array
|
|
VIASH_POSITIONAL_ARGS=''
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-h|--help)
|
|
ViashHelp
|
|
exit
|
|
;;
|
|
---v|---verbose)
|
|
let "VIASH_VERBOSITY=VIASH_VERBOSITY+1"
|
|
shift 1
|
|
;;
|
|
---verbosity)
|
|
VIASH_VERBOSITY="$2"
|
|
shift 2
|
|
;;
|
|
---verbosity=*)
|
|
VIASH_VERBOSITY="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
--version)
|
|
echo "create_eset v0.14.6"
|
|
exit
|
|
;;
|
|
--pDataFile)
|
|
[ -n "$VIASH_PAR_PDATAFILE" ] && ViashError Bad arguments for option \'--pDataFile\': \'$VIASH_PAR_PDATAFILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_PDATAFILE="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --pDataFile. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--pDataFile=*)
|
|
[ -n "$VIASH_PAR_PDATAFILE" ] && ViashError Bad arguments for option \'--pDataFile=*\': \'$VIASH_PAR_PDATAFILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_PDATAFILE=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--fDataFile)
|
|
[ -n "$VIASH_PAR_FDATAFILE" ] && ViashError Bad arguments for option \'--fDataFile\': \'$VIASH_PAR_FDATAFILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_FDATAFILE="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --fDataFile. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--fDataFile=*)
|
|
[ -n "$VIASH_PAR_FDATAFILE" ] && ViashError Bad arguments for option \'--fDataFile=*\': \'$VIASH_PAR_FDATAFILE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_FDATAFILE=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--mappingDir)
|
|
if [ -z "$VIASH_PAR_MAPPINGDIR" ]; then
|
|
VIASH_PAR_MAPPINGDIR="$2"
|
|
else
|
|
VIASH_PAR_MAPPINGDIR="$VIASH_PAR_MAPPINGDIR;""$2"
|
|
fi
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --mappingDir. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--mappingDir=*)
|
|
if [ -z "$VIASH_PAR_MAPPINGDIR" ]; then
|
|
VIASH_PAR_MAPPINGDIR=$(ViashRemoveFlags "$1")
|
|
else
|
|
VIASH_PAR_MAPPINGDIR="$VIASH_PAR_MAPPINGDIR;"$(ViashRemoveFlags "$1")
|
|
fi
|
|
shift 1
|
|
;;
|
|
--poolName)
|
|
[ -n "$VIASH_PAR_POOLNAME" ] && ViashError Bad arguments for option \'--poolName\': \'$VIASH_PAR_POOLNAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_POOLNAME="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --poolName. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--poolName=*)
|
|
[ -n "$VIASH_PAR_POOLNAME" ] && ViashError Bad arguments for option \'--poolName=*\': \'$VIASH_PAR_POOLNAME\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_POOLNAME=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
--output)
|
|
[ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_OUTPUT="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
--output=*)
|
|
[ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
---engine)
|
|
VIASH_ENGINE_ID="$2"
|
|
shift 2
|
|
;;
|
|
---engine=*)
|
|
VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
---setup)
|
|
VIASH_MODE='setup'
|
|
VIASH_SETUP_STRATEGY="$2"
|
|
shift 2
|
|
;;
|
|
---setup=*)
|
|
VIASH_MODE='setup'
|
|
VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")"
|
|
shift 1
|
|
;;
|
|
---dockerfile)
|
|
VIASH_MODE='dockerfile'
|
|
shift 1
|
|
;;
|
|
---docker_run_args)
|
|
VIASH_DOCKER_RUN_ARGS+=("$2")
|
|
shift 2
|
|
;;
|
|
---docker_run_args=*)
|
|
VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")")
|
|
shift 1
|
|
;;
|
|
---docker_image_id)
|
|
VIASH_MODE='docker_image_id'
|
|
shift 1
|
|
;;
|
|
---debug)
|
|
VIASH_MODE='debug'
|
|
shift 1
|
|
;;
|
|
---cpus)
|
|
[ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_CPUS="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
---cpus=*)
|
|
[ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_CPUS=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
---memory)
|
|
[ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_MEMORY="$2"
|
|
[ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1
|
|
shift 2
|
|
;;
|
|
---memory=*)
|
|
[ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1
|
|
VIASH_META_MEMORY=$(ViashRemoveFlags "$1")
|
|
shift 1
|
|
;;
|
|
*) # positional arg or unknown option
|
|
# since the positional args will be eval'd, can we always quote, instead of using ViashQuote
|
|
VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'"
|
|
[[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters.
|
|
shift # past argument
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# parse positional parameters
|
|
eval set -- $VIASH_POSITIONAL_ARGS
|
|
|
|
|
|
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
|
|
VIASH_ENGINE_TYPE='native'
|
|
elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then
|
|
VIASH_ENGINE_TYPE='docker'
|
|
else
|
|
ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# check if docker is installed properly
|
|
ViashDockerInstallationCheck
|
|
|
|
# determine docker image id
|
|
if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then
|
|
VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/htrnaseq/eset/create_eset:v0.14.6'
|
|
fi
|
|
|
|
# print dockerfile
|
|
if [ "$VIASH_MODE" == "dockerfile" ]; then
|
|
ViashDockerfile "$VIASH_ENGINE_ID"
|
|
exit 0
|
|
|
|
elif [ "$VIASH_MODE" == "docker_image_id" ]; then
|
|
echo "$VIASH_DOCKER_IMAGE_ID"
|
|
exit 0
|
|
|
|
# enter docker container
|
|
elif [[ "$VIASH_MODE" == "debug" ]]; then
|
|
VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID"
|
|
ViashNotice "+ $VIASH_CMD"
|
|
eval $VIASH_CMD
|
|
exit
|
|
|
|
# build docker image
|
|
elif [ "$VIASH_MODE" == "setup" ]; then
|
|
ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY"
|
|
ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
|
|
exit 0
|
|
fi
|
|
|
|
# check if docker image exists
|
|
ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild
|
|
ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'ps' 'bash'
|
|
fi
|
|
|
|
# setting computational defaults
|
|
|
|
# helper function for parsing memory strings
|
|
function ViashMemoryAsBytes {
|
|
local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'`
|
|
local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$'
|
|
if [[ $memory =~ $memory_regex ]]; then
|
|
local number=${memory/[^0-9]*/}
|
|
local symbol=${memory/*[0-9]/}
|
|
|
|
case $symbol in
|
|
b) memory_b=$number ;;
|
|
kb|k) memory_b=$(( $number * 1000 )) ;;
|
|
mb|m) memory_b=$(( $number * 1000 * 1000 )) ;;
|
|
gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;;
|
|
tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;;
|
|
pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;;
|
|
kib|ki) memory_b=$(( $number * 1024 )) ;;
|
|
mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;;
|
|
gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;;
|
|
tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;;
|
|
pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;;
|
|
esac
|
|
echo "$memory_b"
|
|
fi
|
|
}
|
|
# compute memory in different units
|
|
if [ ! -z ${VIASH_META_MEMORY+x} ]; then
|
|
VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY`
|
|
# do not define other variables if memory_b is an empty string
|
|
if [ ! -z "$VIASH_META_MEMORY_B" ]; then
|
|
VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 ))
|
|
VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 ))
|
|
VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 ))
|
|
VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 ))
|
|
VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 ))
|
|
VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 ))
|
|
VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 ))
|
|
VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 ))
|
|
else
|
|
# unset memory if string is empty
|
|
unset $VIASH_META_MEMORY_B
|
|
fi
|
|
fi
|
|
# unset nproc if string is empty
|
|
if [ -z "$VIASH_META_CPUS" ]; then
|
|
unset $VIASH_META_CPUS
|
|
fi
|
|
|
|
|
|
# check whether required parameters exist
|
|
if [ -z ${VIASH_PAR_PDATAFILE+x} ]; then
|
|
ViashError '--pDataFile' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_FDATAFILE+x} ]; then
|
|
ViashError '--fDataFile' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_MAPPINGDIR+x} ]; then
|
|
ViashError '--mappingDir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_POOLNAME+x} ]; then
|
|
ViashError '--poolName' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_PAR_OUTPUT+x} ]; then
|
|
ViashError '--output' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_NAME+x} ]; then
|
|
ViashError 'name' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then
|
|
ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then
|
|
ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_EXECUTABLE+x} ]; then
|
|
ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_CONFIG+x} ]; then
|
|
ViashError 'config' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
if [ -z ${VIASH_META_TEMP_DIR+x} ]; then
|
|
ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
|
|
# check whether required files exist
|
|
if [ ! -z "$VIASH_PAR_PDATAFILE" ] && [ ! -e "$VIASH_PAR_PDATAFILE" ]; then
|
|
ViashError "Input file '$VIASH_PAR_PDATAFILE' does not exist."
|
|
exit 1
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_FDATAFILE" ] && [ ! -e "$VIASH_PAR_FDATAFILE" ]; then
|
|
ViashError "Input file '$VIASH_PAR_FDATAFILE' does not exist."
|
|
exit 1
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_MAPPINGDIR" ]; then
|
|
IFS=';'
|
|
set -f
|
|
for file in $VIASH_PAR_MAPPINGDIR; do
|
|
unset IFS
|
|
if [ ! -e "$file" ]; then
|
|
ViashError "Input file '$file' does not exist."
|
|
exit 1
|
|
fi
|
|
done
|
|
set +f
|
|
fi
|
|
|
|
# check whether parameters values are of the right type
|
|
if [[ -n "$VIASH_META_CPUS" ]]; then
|
|
if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_B" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_KB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_MB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_GB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_TB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_PB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then
|
|
if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then
|
|
ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters.
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# create parent directories of output files, if so desired
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then
|
|
mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")"
|
|
fi
|
|
|
|
if [ "$VIASH_ENGINE_ID" == "native" ] ; then
|
|
if [ "$VIASH_MODE" == "run" ]; then
|
|
VIASH_CMD="bash"
|
|
else
|
|
ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# detect volumes from file arguments
|
|
VIASH_CHOWN_VARS=()
|
|
if [ ! -z "$VIASH_PAR_PDATAFILE" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_PDATAFILE")" )
|
|
VIASH_PAR_PDATAFILE=$(ViashDockerAutodetectMount "$VIASH_PAR_PDATAFILE")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_FDATAFILE" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_FDATAFILE")" )
|
|
VIASH_PAR_FDATAFILE=$(ViashDockerAutodetectMount "$VIASH_PAR_FDATAFILE")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_MAPPINGDIR" ]; then
|
|
VIASH_TEST_MAPPINGDIR=()
|
|
IFS=';'
|
|
for var in $VIASH_PAR_MAPPINGDIR; do
|
|
unset IFS
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$var")" )
|
|
var=$(ViashDockerAutodetectMount "$var")
|
|
VIASH_TEST_MAPPINGDIR+=( "$var" )
|
|
done
|
|
VIASH_PAR_MAPPINGDIR=$(IFS=';' ; echo "${VIASH_TEST_MAPPINGDIR[*]}")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" )
|
|
VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT")
|
|
VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" )
|
|
fi
|
|
if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" )
|
|
VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR")
|
|
fi
|
|
if [ ! -z "$VIASH_META_EXECUTABLE" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" )
|
|
VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CONFIG" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" )
|
|
VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG")
|
|
fi
|
|
if [ ! -z "$VIASH_META_TEMP_DIR" ]; then
|
|
VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" )
|
|
VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR")
|
|
fi
|
|
|
|
# get unique mounts
|
|
VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u))
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# change file ownership
|
|
function ViashPerformChown {
|
|
if (( ${#VIASH_CHOWN_VARS[@]} )); then
|
|
set +e
|
|
VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'"
|
|
ViashDebug "+ $VIASH_CMD"
|
|
eval $VIASH_CMD
|
|
set -e
|
|
fi
|
|
}
|
|
trap ViashPerformChown EXIT
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# helper function for filling in extra docker args
|
|
if [ ! -z "$VIASH_META_MEMORY_B" ]; then
|
|
VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CPUS" ]; then
|
|
VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}")
|
|
fi
|
|
fi
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID"
|
|
fi
|
|
|
|
|
|
# set dependency paths
|
|
|
|
|
|
ViashDebug "Running command: $(echo $VIASH_CMD)"
|
|
cat << VIASHEOF | eval $VIASH_CMD
|
|
set -e
|
|
tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-create_eset-XXXXXX").R
|
|
function clean_up {
|
|
rm "\$tempscript"
|
|
}
|
|
function interrupt {
|
|
echo -e "\nCTRL-C Pressed..."
|
|
exit 1
|
|
}
|
|
trap clean_up EXIT
|
|
trap interrupt INT SIGINT
|
|
cat > "\$tempscript" << 'VIASHMAIN'
|
|
library(Biobase)
|
|
library(data.table)
|
|
library(nlcv)
|
|
library(Matrix)
|
|
library(Seurat)
|
|
|
|
### VIASH START
|
|
# The following code has been auto-generated by Viash.
|
|
# treat warnings as errors
|
|
.viash_orig_warn <- options(warn = 2)
|
|
|
|
par <- list(
|
|
"pDataFile" = $( if [ ! -z ${VIASH_PAR_PDATAFILE+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_PDATAFILE" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"fDataFile" = $( if [ ! -z ${VIASH_PAR_FDATAFILE+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_FDATAFILE" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"mappingDir" = $( if [ ! -z ${VIASH_PAR_MAPPINGDIR+x} ]; then echo -n "strsplit('"; echo -n "$VIASH_PAR_MAPPINGDIR" | sed "s#['\\]#\\\\&#g"; echo "', split = ';')[[1]]"; else echo NULL; fi ),
|
|
"poolName" = $( if [ ! -z ${VIASH_PAR_POOLNAME+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_POOLNAME" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"output" = $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo -n "'"; echo -n "$VIASH_PAR_OUTPUT" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi )
|
|
)
|
|
meta <- list(
|
|
"name" = $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo -n "'"; echo -n "$VIASH_META_NAME" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"functionality_name" = $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo -n "'"; echo -n "$VIASH_META_FUNCTIONALITY_NAME" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"resources_dir" = $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo -n "'"; echo -n "$VIASH_META_RESOURCES_DIR" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"executable" = $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo -n "'"; echo -n "$VIASH_META_EXECUTABLE" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"config" = $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo -n "'"; echo -n "$VIASH_META_CONFIG" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"temp_dir" = $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo -n "'"; echo -n "$VIASH_META_TEMP_DIR" | sed "s#['\\]#\\\\&#g"; echo "'"; else echo NULL; fi ),
|
|
"cpus" = $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo -n "as.integer('"; echo -n "$VIASH_META_CPUS" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_b" = $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_B" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_kb" = $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_KB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_mb" = $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_MB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_gb" = $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_GB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_tb" = $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_TB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_pb" = $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_PB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_kib" = $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_KIB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_mib" = $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_MIB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_gib" = $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_GIB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_tib" = $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_TIB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi ),
|
|
"memory_pib" = $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo -n "bit64::as.integer64('"; echo -n "$VIASH_META_MEMORY_PIB" | sed "s#['\\]#\\\\&#g"; echo "')"; else echo NULL; fi )
|
|
)
|
|
dep <- list(
|
|
|
|
)
|
|
|
|
|
|
# restore original warn setting
|
|
options(.viash_orig_warn)
|
|
rm(.viash_orig_warn)
|
|
|
|
### VIASH END
|
|
|
|
|
|
Read10X <- function(data_dir = NULL, gene_column = 2, unique_features = TRUE) {
|
|
full.data <- list()
|
|
for (i in seq_along(along.with = data_dir)) {
|
|
run <- data_dir[i]
|
|
if (!dir.exists(paths = run)) {
|
|
stop("Directory provided does not exist")
|
|
}
|
|
barcode.loc <- file.path(run, "barcodes.tsv")
|
|
gene.loc <- file.path(run, "features.tsv")
|
|
features.loc <- file.path(run, "features.tsv.gz")
|
|
matrix.loc <- file.path(run, "matrix.mtx")
|
|
pre_ver_3 <- file.exists(gene.loc)
|
|
if (!pre_ver_3) {
|
|
addgz <- function(s) {
|
|
return(paste0(s, ".gz"))
|
|
}
|
|
barcode.loc <- addgz(s = barcode.loc)
|
|
matrix.loc <- addgz(s = matrix.loc)
|
|
}
|
|
if (!file.exists(barcode.loc)) {
|
|
stop("Barcode file missing")
|
|
}
|
|
if (!pre_ver_3 && !file.exists(features.loc)) {
|
|
stop("Gene name or features file missing")
|
|
}
|
|
if (!file.exists(matrix.loc)) {
|
|
stop("Expression matrix file missing")
|
|
}
|
|
data <- readMM(file = matrix.loc)
|
|
cell.names <- readLines(barcode.loc)
|
|
if (all(grepl(pattern = "\\\\-1\$", x = cell.names))) {
|
|
cell.names <- as.vector(x = as.character(x = sapply(X = cell.names,
|
|
FUN = ExtractField, field = 1, delim = "-")))
|
|
}
|
|
if (is.null(x = names(x = data_dir))) {
|
|
if (i < 2) {
|
|
colnames(x = data) <- cell.names
|
|
}
|
|
else {
|
|
colnames(x = data) <- paste0(i, "_", cell.names)
|
|
}
|
|
}
|
|
else {
|
|
colnames(x = data) <- paste0(names(x = data_dir)[i],
|
|
"_", cell.names)
|
|
}
|
|
feature.names <- read.delim(file = ifelse(test = pre_ver_3,
|
|
yes = gene.loc, no = features.loc), header = FALSE,
|
|
stringsAsFactors = FALSE)
|
|
if (any(is.na(x = feature.names[, gene_column]))) {
|
|
warning("Some features names are NA. Replacing NA names with ID from the opposite column requested",
|
|
call. = FALSE, immediate. = TRUE)
|
|
na.features <- which(x = is.na(x = feature.names[,
|
|
gene_column]))
|
|
replacement.column <- ifelse(test = gene_column ==
|
|
2, yes = 1, no = 2)
|
|
feature.names[na.features, gene_column] <- feature.names[na.features,
|
|
replacement.column]
|
|
}
|
|
if (unique_features) {
|
|
fcols = ncol(x = feature.names)
|
|
if (fcols < gene_column) {
|
|
stop(paste0("gene_column was set to ", gene_column,
|
|
" but feature.tsv.gz (or genes.tsv) only has ",
|
|
fcols, " columns.", " Try setting the gene_column ",
|
|
"argument to a value <= to ",
|
|
fcols, "."))
|
|
}
|
|
rownames(x = data) <- make.unique(names = feature.names[,
|
|
gene_column])
|
|
}
|
|
if (ncol(x = feature.names) > 2) {
|
|
data_types <- factor(x = feature.names\$V3)
|
|
lvls <- levels(x = data_types)
|
|
if (length(x = lvls) > 1 && length(x = full.data) == 0) {
|
|
message(paste0("10X data contains more than one type and is ",
|
|
"being returned as a list containing matrices ",
|
|
"of each type."))
|
|
}
|
|
expr_name <- "Gene Expression"
|
|
if (expr_name %in% lvls) {
|
|
lvls <- c(expr_name, lvls[-which(x = lvls ==
|
|
expr_name)])
|
|
}
|
|
data <- lapply(X = lvls, FUN = function(l) {
|
|
return(data[data_types == l, , drop = FALSE])
|
|
})
|
|
names(x = data) <- lvls
|
|
} else {
|
|
data <- list(data)
|
|
}
|
|
full.data[[length(x = full.data) + 1]] <- data
|
|
}
|
|
list_of_data <- list()
|
|
for (j in 1:length(x = full.data[[1]])) {
|
|
list_of_data[[j]] <- do.call(cbind, lapply(X = full.data,
|
|
FUN = \`[[\`, j))
|
|
list_of_data[[j]] <- as(object = list_of_data[[j]], Class = "CsparseMatrix")
|
|
}
|
|
names(x = list_of_data) <- names(x = full.data[[1]])
|
|
if (length(x = list_of_data) == 1) {
|
|
return(list_of_data[[1]])
|
|
} else {
|
|
return(list_of_data)
|
|
}
|
|
}
|
|
|
|
match_features <- function(exprs_matrix, fdata) {
|
|
|
|
identical_features <- all(rownames(exprs_matrix) == rownames(fdata))
|
|
|
|
if (nrow(exprs_matrix) != nrow(fdata) || !identical_features) {
|
|
message(paste0("Features in 'fData' and expression matrix differ. ",
|
|
"Only matching features are returned."))
|
|
}
|
|
|
|
features <- intersect(rownames(exprs_matrix), rownames(fdata))
|
|
exprs_matrix <- exprs_matrix[which(rownames(exprs_matrix) %in% features), ]
|
|
fdata <- fdata[which(rownames(fdata) %in% features), ]
|
|
|
|
fdata[, seq_len(ncol(fdata))] <- lapply(fdata[, seq_len(ncol(fdata)), drop = FALSE], as.character)
|
|
# order features in exprs mat according to fdata
|
|
exprs_matrix <- exprs_matrix[match(rownames(fdata), rownames(exprs_matrix)), ]
|
|
|
|
list(exprs_matrix = exprs_matrix, fdata = fdata)
|
|
|
|
}
|
|
|
|
|
|
create_pdata <- function(sample_file, pool_name, barcodes) {
|
|
cols_to_remove <- c("SampleFileName", "Output", "Measure", "Strandedness")
|
|
pData <- sample_file[, !colnames(sample_file) %in% cols_to_remove,
|
|
drop = FALSE]
|
|
rownames(pData) <- lapply(sample_file\$WellBC,
|
|
\\(x) paste(pool_name, x, sep = "_"))
|
|
# pData[, ] <- lapply(pData, as.factor)
|
|
pData\$PoolName <- pool_name
|
|
pData <- pData[match(barcodes, pData\$WellBC), ]
|
|
return(pData)
|
|
}
|
|
|
|
check_sample_file <- function(mapping_dir, sample_file){
|
|
|
|
message("Checking sample annotation:")
|
|
|
|
requireNamespace("tools")
|
|
mapping_dir <- unlist(lapply(mapping_dir, function(x) {
|
|
if (!dir.exists(x)) {
|
|
stop(sprintf(paste0("Could not find directory ",
|
|
"provided in 'mappingDir' argument (%s)."), x))
|
|
}
|
|
tools::file_path_as_absolute(x)
|
|
}))
|
|
|
|
|
|
# additional check for STARsolo
|
|
check_STARsolo_output <- function(x) {
|
|
files <- c("barcodes.tsv", "features.tsv", "matrix.mtx")
|
|
test <- list.files(x) %in% c(files, paste0(files, ".gz"))
|
|
length(test) != 0 && all(test)
|
|
}
|
|
|
|
|
|
if (!"WellBC" %in% colnames(sample_file)) {
|
|
stop(paste0("STARsolo output is used. The sample annotation must ",
|
|
"contain 'WellBC' column providing cell barcodes."))
|
|
}
|
|
|
|
mapping_dir <- unique(mapping_dir)
|
|
all_STARsolo_files_present <- all(
|
|
unlist(
|
|
lapply(mapping_dir, function(x) {
|
|
check_STARsolo_output(x)
|
|
})
|
|
)
|
|
)
|
|
if (!all_STARsolo_files_present) {
|
|
stop(paste0("Could not find files: 'barcodes', 'features' and 'matrix'",
|
|
" for STARsolo output. Please check 'mappingDir' argument."))
|
|
}
|
|
|
|
message("- 'SampleFileName' column - OK")
|
|
|
|
|
|
|
|
list(sample_expression_files = mapping_dir)
|
|
}
|
|
|
|
create_exprs_matrix <- function(exprs_matrix_path, exprs_file_paths,
|
|
output, measure, col_names, cell_barcodes) {
|
|
|
|
read_matrix <- Read10X(data_dir = exprs_file_paths, gene_column = 1)
|
|
# keep index of feature names containing "_" because Seurat
|
|
#changes them to "-" and they no longer match with fdata[, "gene_id"]
|
|
idx <- grep("_", rownames(read_matrix))
|
|
|
|
requireNamespace("Seurat")
|
|
seurat_object <- Seurat::CreateSeuratObject(counts = read_matrix)
|
|
|
|
exprs_matrix <- as.matrix(seurat_object[['RNA']]\$counts)
|
|
# replace "-" with "_" for features with "_"
|
|
# before converting to Seurat object
|
|
rownames(exprs_matrix)[idx] <- gsub("-", "_", rownames(exprs_matrix)[idx])
|
|
requireNamespace("stringr")
|
|
exprs_matrix <- exprs_matrix[, stringr::str_detect(colnames(exprs_matrix),
|
|
paste(cell_barcodes, collapse = "|"))]
|
|
|
|
|
|
# check if rownames are ENSEMBL and remove version suffix
|
|
isENSEMBL <- all(grepl("ENS", rownames(exprs_matrix)))
|
|
if (isENSEMBL) {
|
|
# do not use gsub("(.+)[.]\\\\d+", "\\\\1", rownames(exprs_matrix)),
|
|
# so that ENS000000.1_PAR_Y can be kept
|
|
rownames(exprs_matrix) <- gsub("\\\\.\\\\d+\$", "", rownames(exprs_matrix))
|
|
}
|
|
|
|
|
|
colnames(exprs_matrix) <- col_names
|
|
|
|
exprs_matrix
|
|
}
|
|
|
|
create_eset <- function(feature_annotation_path,
|
|
sample_annotation_path,
|
|
mapping_dir,
|
|
barcodes,
|
|
output_path,
|
|
pool_name,
|
|
exprs_matrix_path = NULL,
|
|
path = NULL,
|
|
add_eset_annotation = NULL) {
|
|
if (!file.exists(feature_annotation_path)) {
|
|
stop("Could not find feature annotation at '", feature_annotation_path, "'")
|
|
}
|
|
|
|
if (!file.exists(sample_annotation_path)) {
|
|
stop("Could not find sample annotation at '", sample_annotation_path, "'")
|
|
}
|
|
|
|
if(!is.null(exprs_matrix_path)) {
|
|
if(!file.exists(exprs_matrix_path)) {
|
|
stop("Could not find expression matrix at '", exprs_matrix_path, "'")
|
|
}
|
|
}
|
|
|
|
if(!is.null(path)) {
|
|
if(!dir.exists(path)) {
|
|
stop("Provided 'path': '", path, "' does not exist.")
|
|
}
|
|
}
|
|
|
|
##### Import annotation files #####
|
|
message("Importing feature annotation")
|
|
fdata_file <- read.table(feature_annotation_path, header = TRUE,
|
|
sep = "\\t", quote = "\\"",
|
|
comment.char = "", stringsAsFactors = FALSE)
|
|
|
|
# for backwards compatibility
|
|
if("ENSEMBL" %in% colnames(fdata_file) && !all(grepl("ENS", fdata_file[, "ENSEMBL"])) & !"gene_id" %in% colnames(fdata_file)) {
|
|
colnames(fdata_file)[which(colnames(fdata_file) == "ENSEMBL")] <- "gene_id"
|
|
}
|
|
|
|
# Check gene annotation
|
|
if(!"gene_id" %in% colnames(fdata_file))
|
|
stop("'gene_id' column with unique feature identifiers must be present in 'feature_annotation_path'.")
|
|
|
|
# check if duplicated ids are present
|
|
if(any(duplicated(fdata_file\$gene_id)))
|
|
stop("Duplicated features ids are not allowed. Please check the 'gene_id' column in 'feature_annotation_path'.")
|
|
|
|
message("Importing sample annotation")
|
|
sample_file <- read.table(sample_annotation_path, header = TRUE,
|
|
sep = "\\t", quote = "\\"",
|
|
comment.char = "", stringsAsFactors = FALSE)
|
|
# Check sample annotation
|
|
check_sample_file_list <- check_sample_file(mapping_dir = mapping_dir,
|
|
sample_file = sample_file)
|
|
output <- "STARsolo"
|
|
measure <- "counts"
|
|
sample_expression_files <- check_sample_file_list\$sample_expression_files
|
|
|
|
##### Create phenodata #####
|
|
pdata_eset <- create_pdata(sample_file = sample_file, pool_name = pool_name,
|
|
barcodes = barcodes)
|
|
|
|
##### Create expression matrix #####
|
|
message("Creating expression matrix")
|
|
|
|
exprs_matrix_eset <- create_exprs_matrix(
|
|
exprs_matrix_path = exprs_matrix_path,
|
|
exprs_file_paths = sample_expression_files,
|
|
output = output,
|
|
measure = measure,
|
|
col_names = rownames(pdata_eset),
|
|
cell_barcodes = barcodes
|
|
)
|
|
|
|
|
|
##### Create featuredata #####
|
|
message("Creating feature data")
|
|
|
|
fdata_eset <- fdata_file
|
|
rownames(fdata_eset) <- fdata_eset[, "gene_id"]
|
|
|
|
# intersect features between exprs matrix and fdata
|
|
feature_files <- match_features(exprs_matrix = exprs_matrix_eset,
|
|
fdata = fdata_eset)
|
|
|
|
fdata_eset <- feature_files\$fdata
|
|
exprs_matrix_eset <- feature_files\$exprs_matrix
|
|
|
|
##### Create eSet #####
|
|
message("Creating eset")
|
|
|
|
if (nrow(pdata_eset) != ncol(exprs_matrix_eset)) {
|
|
stop("nrow(pData) and ncol(exprsMatrix) differ")
|
|
}
|
|
|
|
if (nrow(fdata_eset) != nrow(exprs_matrix_eset)) {
|
|
stop("nrow(fData) and nrow(exprsMatrix) differ")
|
|
}
|
|
|
|
if (!all(rownames(pdata_eset) == colnames(exprs_matrix_eset))) {
|
|
stop("rownames(pData) and colnames(exprsMatrix) differ")
|
|
}
|
|
|
|
if (!all(rownames(fdata_eset) == rownames(exprs_matrix_eset))) {
|
|
stop("rownames(fData) and rownames(exprsMatrix) differ")
|
|
}
|
|
|
|
if (!inherits(exprs_matrix_eset, "matrix")) {
|
|
stop("exprsMatrix must be of class 'matrix'")
|
|
}
|
|
|
|
|
|
|
|
additional_info <- paste0("Additional information about eSet \\n",
|
|
" Expression matrix created from ",
|
|
output, " output. \\n",
|
|
" Expression matrix contains non-transformed ",
|
|
ifelse(output %in% c("STAR", "STARsolo"),
|
|
"counts",
|
|
ifelse(measure == "expected_count",
|
|
"counts", measure)), ".")
|
|
|
|
|
|
if (isTRUE(!is.null(add_eset_annotation) &
|
|
is.character(add_eset_annotation))) {
|
|
additional_info <- paste0(additional_info, "\\n", " ", add_eset_annotation)
|
|
}
|
|
|
|
fdata_eset <- new("AnnotatedDataFrame", data = fdata_eset)
|
|
pdata_eset <- new("AnnotatedDataFrame", data = pdata_eset)
|
|
|
|
requireNamespace("Biobase")
|
|
eset <- Biobase::ExpressionSet(assayData = exprs_matrix_eset,
|
|
phenoData = pdata_eset,
|
|
featureData = fdata_eset,
|
|
annotation = additional_info)
|
|
|
|
eset <- eset[, colSums(exprs(eset)) != 0]
|
|
|
|
if (nrow(eset) == 0) {
|
|
stop("Count matrix does not seem to contain any features.")
|
|
}
|
|
|
|
if (ncol(eset) == 0) {
|
|
stop("Count matrix does not seem to contain any samples.")
|
|
}
|
|
|
|
|
|
saveRDS(eset, file = output_path)
|
|
|
|
message(paste0("eset created succesfully for ", ncol(eset),
|
|
" samples and ", nrow(eset),
|
|
" genes and saved at ", output_path, "."))
|
|
|
|
eset
|
|
}
|
|
|
|
|
|
p_data_file <- par\$pDataFile
|
|
f_data_file <- par\$fDataFile
|
|
pool_name <- par\$poolName
|
|
mapping_dir <- lapply(par\$mappingDir,
|
|
\\(x) file.path(x, "Solo.out", "Gene", "raw"))
|
|
|
|
get_barcode_from_mapping_dir <- function(raw_dir) {
|
|
barcodes_file <- file.path(raw_dir, "barcodes.tsv")
|
|
if (!file.exists(barcodes_file)) {
|
|
stop(paste0("Expected the 'Solo.out/Gene/raw' directory at ",
|
|
raw_dir, " to contain a 'barcodes.tsv' file."))
|
|
}
|
|
barcodes <- readLines(barcodes_file)
|
|
if (length(barcodes) != 1) {
|
|
stop(paste0("A single STAR Solo folder should only have ",
|
|
"mapped one (1) barcode, but found '",
|
|
length(barcodes), "'for mapping directory ", raw_dir))
|
|
}
|
|
return(barcodes)
|
|
}
|
|
|
|
barcodes <- lapply(mapping_dir, get_barcode_from_mapping_dir)
|
|
|
|
print(paste0("mappingDir: ", mapping_dir))
|
|
print(paste0("pDataFile: ", p_data_file))
|
|
print(paste0("fDataFile: ", f_data_file))
|
|
print(paste0("poolName: ", pool_name))
|
|
print(paste0("barcodes: ", barcodes))
|
|
|
|
|
|
|
|
# CREATE ESET WITH RAW UMI COUNTS
|
|
|
|
eset <- create_eset(feature_annotation_path = f_data_file,
|
|
sample_annotation_path = p_data_file,
|
|
mapping_dir = mapping_dir,
|
|
barcodes = barcodes,
|
|
output_path = par\$output,
|
|
pool_name = pool_name,
|
|
path = NULL,
|
|
exprs_matrix_path = NULL)
|
|
VIASHMAIN
|
|
Rscript "\$tempscript" &
|
|
wait "\$!"
|
|
|
|
VIASHEOF
|
|
|
|
|
|
if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then
|
|
# strip viash automount from file paths
|
|
|
|
if [ ! -z "$VIASH_PAR_PDATAFILE" ]; then
|
|
VIASH_PAR_PDATAFILE=$(ViashDockerStripAutomount "$VIASH_PAR_PDATAFILE")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_FDATAFILE" ]; then
|
|
VIASH_PAR_FDATAFILE=$(ViashDockerStripAutomount "$VIASH_PAR_FDATAFILE")
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_MAPPINGDIR" ]; then
|
|
unset VIASH_TEST_MAPPINGDIR
|
|
IFS=';'
|
|
for var in $VIASH_PAR_MAPPINGDIR; do
|
|
unset IFS
|
|
if [ -z "$VIASH_TEST_MAPPINGDIR" ]; then
|
|
VIASH_TEST_MAPPINGDIR="$(ViashDockerStripAutomount "$var")"
|
|
else
|
|
VIASH_TEST_MAPPINGDIR="$VIASH_TEST_MAPPINGDIR;""$(ViashDockerStripAutomount "$var")"
|
|
fi
|
|
done
|
|
VIASH_PAR_MAPPINGDIR="$VIASH_TEST_MAPPINGDIR"
|
|
fi
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ]; then
|
|
VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT")
|
|
fi
|
|
if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then
|
|
VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR")
|
|
fi
|
|
if [ ! -z "$VIASH_META_EXECUTABLE" ]; then
|
|
VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE")
|
|
fi
|
|
if [ ! -z "$VIASH_META_CONFIG" ]; then
|
|
VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG")
|
|
fi
|
|
if [ ! -z "$VIASH_META_TEMP_DIR" ]; then
|
|
VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR")
|
|
fi
|
|
fi
|
|
|
|
|
|
# check whether required files exist
|
|
if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then
|
|
ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist."
|
|
exit 1
|
|
fi
|
|
|
|
|
|
exit 0
|