From 448ee2c64822fa0fcf99d21d94277dd637c19f52 Mon Sep 17 00:00:00 2001 From: CI Date: Tue, 10 Jun 2025 14:50:16 +0000 Subject: [PATCH] Build branch check-resources with version check-resources (cf2c8d1) Build pipeline: viash-hub.toolbox.check-resources-vph6n Source commit: https://github.com/viash-hub/toolbox/commit/cf2c8d1810fd106dbf1c36a5d25e885f8285d541 Source message: add mock tests --- src/check_resources/config.vsh.yaml | 32 ++-- src/check_resources/script.sh | 22 ++- src/check_resources/test.sh | 92 +++++++++ .../check_resources/.config.vsh.yaml | 40 ++-- .../check_resources/check_resources | 142 ++++++++------ .../nextflow/check_resources/.config.vsh.yaml | 40 ++-- target/nextflow/check_resources/main.nf | 76 +++++--- .../check_resources/nextflow_schema.json | 181 +++++++----------- 8 files changed, 379 insertions(+), 246 deletions(-) create mode 100644 src/check_resources/test.sh diff --git a/src/check_resources/config.vsh.yaml b/src/check_resources/config.vsh.yaml index 804db0d..beae54a 100644 --- a/src/check_resources/config.vsh.yaml +++ b/src/check_resources/config.vsh.yaml @@ -4,20 +4,20 @@ description: Check for available resources on the system. argument_groups: - name: Inputs arguments: - - name: --input - type: file - direction: input - description: Input yaml file containing input parameters of a workflow. - required: true - default: input.yaml - - - name: Options - arguments: - - name: --req_disk_space + - name: --tmp_space_required type: integer - description: Required disk space percentage. - required: true - default: 20 + description: Temporary space required in MB. + required: false + default: 1000 + - name: --publish_space_required + type: integer + description: Publish space required in MB. + required: false + default: 500 + - name: --publish_dir + type: file + description: Directory where workflow outputs will be published. + required: false - name: Outputs arguments: @@ -25,13 +25,17 @@ argument_groups: type: file direction: output description: Output file containing system resources details. - required: true + required: false default: output.txt resources: - type: bash_script path: script.sh +test_resources: + - type: bash_script + path: test.sh + engines: - type: docker image: bash:latest diff --git a/src/check_resources/script.sh b/src/check_resources/script.sh index 04c6f3e..d58530d 100644 --- a/src/check_resources/script.sh +++ b/src/check_resources/script.sh @@ -1,7 +1,19 @@ -usage_percentage=$(df -h "$(pwd)" | awk 'NR==2 {print $5}' | tr -d '%') -df -h "$(pwd)" > "$par_output" +#!/bin/bash -max_allowed_usage=$((100-par_req_disk_space)) -if [ "$usage_percentage" -gt "$max_allowed_usage" ]; then - echo "WARNING: Disk usage is at ${usage_percentage}%, exceeding maximum allowed usage of ${max_allowed_usage}%" +# Check temporary directory space +tmp_avail_kb=$(df -k "$meta_temp_dir" | awk 'NR==2 {print $4}') +tmp_avail_mb=$((tmp_avail_kb / 1024)) +echo -e "\nTemporary directory ($meta_temp_dir) available space: ${tmp_avail_mb}MB" >> "$par_output" + +if [ "$tmp_avail_mb" -lt "$par_tmp_space_required" ]; then + echo "WARNING: Available temporary space (${tmp_avail_mb}MB) is less than required (${par_tmp_space_required}MB)" | tee -a "$par_output" +fi + +# Check publish directory space if specified +publish_avail_kb=$(df -k "$par_publish_dir" | awk 'NR==2 {print $4}') +publish_avail_mb=$((publish_avail_kb / 1024)) +echo -e "\nPublish directory ($par_publish_dir) available space: ${publish_avail_mb}MB" >> "$par_output" + +if [ "$publish_avail_mb" -lt "$par_publish_space_required" ]; then + echo "WARNING: Available publish space (${publish_avail_mb}MB) is less than required (${par_publish_space_required}MB)" | tee -a "$par_output" fi \ No newline at end of file diff --git a/src/check_resources/test.sh b/src/check_resources/test.sh new file mode 100644 index 0000000..3f2558f --- /dev/null +++ b/src/check_resources/test.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -eo pipefail + +TMPDIR=$(mktemp -d "$meta_temp_dir/$meta_name-XXXXXX") +function clean_up { + [[ -d "$TMPDIR" ]] && rm -rf "$TMPDIR" +} +trap clean_up EXIT + +# Create test directories +mkdir -p "$TMPDIR/publish" + +echo "Test 1: Both directories have enough space" +# Define normal space df function +df() { + echo "Filesystem 1K-blocks Used Available Use% Mounted on" + echo "ext4 20485760 5000000 15485760 25% $2" +} +# Export the function +export -f df +# Run the component +"$meta_executable" \ + --publish_dir "$TMPDIR/publish" \ + --tmp_space_required "500" \ + --publish_space_required "300" \ + --output "$TMPDIR/output1.txt" + +# Verify Test 1 - should have no warnings +if grep -q "WARNING:" "$TMPDIR/output1.txt"; then + echo "FAIL: Unexpected warning in normal space test" +else + echo "PASS: No warnings with sufficient space" +fi + +echo "Test 2: Temporary directory doesn't have enough space" +# Define low temp space df function +df() { + if [[ "$2" == "$meta_temp_dir" ]]; then + echo "Filesystem 1K-blocks Used Available Use% Mounted on" + echo "tmpfs 10485760 5000000 921600 50% $2" + else + echo "Filesystem 1K-blocks Used Available Use% Mounted on" + echo "ext4 20485760 5000000 15485760 25% $2" + fi +} +# Export the function +export -f df +# Run the component +"$meta_executable" \ + --publish_dir "$TMPDIR/publish" \ + --tmp_space_required "1000" \ + --publish_space_required "300" \ + --output "$TMPDIR/output2.txt" + +# Verify Test 2 +if grep -q "WARNING: Available temporary space" "$TMPDIR/output2.txt"; then + echo "PASS: Low temp space warning detected" +else + echo "FAIL: No warning detected for low temp space" +fi + +echo "Test 3: Publish directory doesn't have enough space" +# Define low publish space df function +df() { + if [[ "$2" =~ /publish ]]; then + echo "Filesystem 1K-blocks Used Available Use% Mounted on" + echo "ext4 20485760 16384000 409600 80% $2" + else + echo "Filesystem 1K-blocks Used Available Use% Mounted on" + echo "ext4 20485760 5000000 15485760 25% $2" + fi +} +# Export the function +export -f df +# Run the component +"$meta_executable" \ + --publish_dir "$TMPDIR/publish" \ + --tmp_space_required "500" \ + --publish_space_required "500" \ + --output "$TMPDIR/output3.txt" + +# Verify Test 3 +if grep -q "WARNING: Available publish space" "$TMPDIR/output3.txt"; then + echo "PASS: Low publish space warning detected" +else + echo "FAIL: No warning detected for low publish space" +fi + +echo "All tests completed" + +exit 0 \ No newline at end of file diff --git a/target/executable/check_resources/.config.vsh.yaml b/target/executable/check_resources/.config.vsh.yaml index ef998f7..a962c72 100644 --- a/target/executable/check_resources/.config.vsh.yaml +++ b/target/executable/check_resources/.config.vsh.yaml @@ -3,27 +3,33 @@ version: "check-resources" argument_groups: - name: "Inputs" arguments: - - type: "file" - name: "--input" - description: "Input yaml file containing input parameters of a workflow." + - type: "integer" + name: "--tmp_space_required" + description: "Temporary space required in MB." info: null default: - - "input.yaml" - must_exist: true - create_parent: true - required: true + - 1000 + required: false direction: "input" multiple: false multiple_sep: ";" -- name: "Options" - arguments: - type: "integer" - name: "--req_disk_space" - description: "Required disk space percentage." + name: "--publish_space_required" + description: "Publish space required in MB." info: null default: - - 20 - required: true + - 500 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--publish_dir" + description: "Directory where workflow outputs will be published." + info: null + must_exist: true + create_parent: true + required: false direction: "input" multiple: false multiple_sep: ";" @@ -37,7 +43,7 @@ argument_groups: - "output.txt" must_exist: true create_parent: true - required: true + required: false direction: "output" multiple: false multiple_sep: ";" @@ -46,6 +52,10 @@ resources: path: "script.sh" is_executable: true description: "Check for available resources on the system." +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true info: null status: "enabled" requirements: @@ -137,7 +147,7 @@ build_info: output: "target/executable/check_resources" executable: "target/executable/check_resources/check_resources" viash_version: "0.9.0" - git_commit: "bd253a050d688b378752759d8c5293f4e333ee34" + git_commit: "cf2c8d1810fd106dbf1c36a5d25e885f8285d541" git_remote: "https://github.com/viash-hub/toolbox" package_config: name: "toolbox" diff --git a/target/executable/check_resources/check_resources b/target/executable/check_resources/check_resources index 71b2614..e1ad546 100755 --- a/target/executable/check_resources/check_resources +++ b/target/executable/check_resources/check_resources @@ -176,20 +176,23 @@ function ViashHelp { echo "Check for available resources on the system." echo "" echo "Inputs:" - echo " --input" - echo " type: file, required parameter, file must exist" - echo " default: input.yaml" - echo " Input yaml file containing input parameters of a workflow." + echo " --tmp_space_required" + echo " type: integer" + echo " default: 1000" + echo " Temporary space required in MB." echo "" - echo "Options:" - echo " --req_disk_space" - echo " type: integer, required parameter" - echo " default: 20" - echo " Required disk space percentage." + echo " --publish_space_required" + echo " type: integer" + echo " default: 500" + echo " Publish space required in MB." + echo "" + echo " --publish_dir" + echo " type: file, file must exist" + echo " Directory where workflow outputs will be published." echo "" echo "Outputs:" echo " --output" - echo " type: file, required parameter, output, file must exist" + echo " type: file, output, file must exist" echo " default: output.txt" echo " Output file containing system resources details." } @@ -470,9 +473,9 @@ function ViashDockerfile { FROM bash:latest ENTRYPOINT [] LABEL org.opencontainers.image.description="Companion container for running component check_resources" -LABEL org.opencontainers.image.created="2025-04-23T13:05:55Z" +LABEL org.opencontainers.image.created="2025-06-10T14:43:12Z" LABEL org.opencontainers.image.source="https://github.com/viash-hub/toolbox" -LABEL org.opencontainers.image.revision="bd253a050d688b378752759d8c5293f4e333ee34" +LABEL org.opencontainers.image.revision="cf2c8d1810fd106dbf1c36a5d25e885f8285d541" LABEL org.opencontainers.image.version="check-resources" VIASHDOCKER @@ -612,26 +615,37 @@ while [[ $# -gt 0 ]]; do echo "check_resources check-resources" exit ;; - --input) - [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_INPUT="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + --tmp_space_required) + [ -n "$VIASH_PAR_TMP_SPACE_REQUIRED" ] && ViashError Bad arguments for option \'--tmp_space_required\': \'$VIASH_PAR_TMP_SPACE_REQUIRED\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_TMP_SPACE_REQUIRED="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --tmp_space_required. Use "--help" to get more information on the parameters. && exit 1 shift 2 ;; - --input=*) - [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + --tmp_space_required=*) + [ -n "$VIASH_PAR_TMP_SPACE_REQUIRED" ] && ViashError Bad arguments for option \'--tmp_space_required=*\': \'$VIASH_PAR_TMP_SPACE_REQUIRED\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_TMP_SPACE_REQUIRED=$(ViashRemoveFlags "$1") shift 1 ;; - --req_disk_space) - [ -n "$VIASH_PAR_REQ_DISK_SPACE" ] && ViashError Bad arguments for option \'--req_disk_space\': \'$VIASH_PAR_REQ_DISK_SPACE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_REQ_DISK_SPACE="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --req_disk_space. Use "--help" to get more information on the parameters. && exit 1 + --publish_space_required) + [ -n "$VIASH_PAR_PUBLISH_SPACE_REQUIRED" ] && ViashError Bad arguments for option \'--publish_space_required\': \'$VIASH_PAR_PUBLISH_SPACE_REQUIRED\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PUBLISH_SPACE_REQUIRED="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --publish_space_required. Use "--help" to get more information on the parameters. && exit 1 shift 2 ;; - --req_disk_space=*) - [ -n "$VIASH_PAR_REQ_DISK_SPACE" ] && ViashError Bad arguments for option \'--req_disk_space=*\': \'$VIASH_PAR_REQ_DISK_SPACE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_REQ_DISK_SPACE=$(ViashRemoveFlags "$1") + --publish_space_required=*) + [ -n "$VIASH_PAR_PUBLISH_SPACE_REQUIRED" ] && ViashError Bad arguments for option \'--publish_space_required=*\': \'$VIASH_PAR_PUBLISH_SPACE_REQUIRED\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PUBLISH_SPACE_REQUIRED=$(ViashRemoveFlags "$1") + shift 1 + ;; + --publish_dir) + [ -n "$VIASH_PAR_PUBLISH_DIR" ] && ViashError Bad arguments for option \'--publish_dir\': \'$VIASH_PAR_PUBLISH_DIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PUBLISH_DIR="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --publish_dir. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --publish_dir=*) + [ -n "$VIASH_PAR_PUBLISH_DIR" ] && ViashError Bad arguments for option \'--publish_dir=*\': \'$VIASH_PAR_PUBLISH_DIR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_PUBLISH_DIR=$(ViashRemoveFlags "$1") shift 1 ;; --output) @@ -817,18 +831,6 @@ fi # check whether required parameters exist -if [ -z ${VIASH_PAR_INPUT+x} ]; then - ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_PAR_REQ_DISK_SPACE+x} ]; then - ViashError '--req_disk_space' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi -if [ -z ${VIASH_PAR_OUTPUT+x} ]; then - ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi if [ -z ${VIASH_META_NAME+x} ]; then ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. exit 1 @@ -854,16 +856,33 @@ if [ -z ${VIASH_META_TEMP_DIR+x} ]; then exit 1 fi +# filling in defaults +if [ -z ${VIASH_PAR_TMP_SPACE_REQUIRED+x} ]; then + VIASH_PAR_TMP_SPACE_REQUIRED="1000" +fi +if [ -z ${VIASH_PAR_PUBLISH_SPACE_REQUIRED+x} ]; then + VIASH_PAR_PUBLISH_SPACE_REQUIRED="500" +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + VIASH_PAR_OUTPUT="output.txt" +fi + # check whether required files exist -if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then - ViashError "Input file '$VIASH_PAR_INPUT' does not exist." +if [ ! -z "$VIASH_PAR_PUBLISH_DIR" ] && [ ! -e "$VIASH_PAR_PUBLISH_DIR" ]; then + ViashError "Input file '$VIASH_PAR_PUBLISH_DIR' does not exist." exit 1 fi # check whether parameters values are of the right type -if [[ -n "$VIASH_PAR_REQ_DISK_SPACE" ]]; then - if ! [[ "$VIASH_PAR_REQ_DISK_SPACE" =~ ^[-+]?[0-9]+$ ]]; then - ViashError '--req_disk_space' has to be an integer. Use "--help" to get more information on the parameters. +if [[ -n "$VIASH_PAR_TMP_SPACE_REQUIRED" ]]; then + if ! [[ "$VIASH_PAR_TMP_SPACE_REQUIRED" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--tmp_space_required' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_PUBLISH_SPACE_REQUIRED" ]]; then + if ! [[ "$VIASH_PAR_PUBLISH_SPACE_REQUIRED" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--publish_space_required' has to be an integer. Use "--help" to get more information on the parameters. exit 1 fi fi @@ -957,9 +976,9 @@ fi if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # detect volumes from file arguments VIASH_CHOWN_VARS=() -if [ ! -z "$VIASH_PAR_INPUT" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) - VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +if [ ! -z "$VIASH_PAR_PUBLISH_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_PUBLISH_DIR")" ) + VIASH_PAR_PUBLISH_DIR=$(ViashDockerAutodetectMount "$VIASH_PAR_PUBLISH_DIR") fi if [ ! -z "$VIASH_PAR_OUTPUT" ]; then VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) @@ -1035,8 +1054,9 @@ trap interrupt INT SIGINT cat > "\$tempscript" << 'VIASHMAIN' ## VIASH START # The following code has been auto-generated by Viash. -$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) -$( if [ ! -z ${VIASH_PAR_REQ_DISK_SPACE+x} ]; then echo "${VIASH_PAR_REQ_DISK_SPACE}" | sed "s#'#'\"'\"'#g;s#.*#par_req_disk_space='&'#" ; else echo "# par_req_disk_space="; fi ) +$( if [ ! -z ${VIASH_PAR_TMP_SPACE_REQUIRED+x} ]; then echo "${VIASH_PAR_TMP_SPACE_REQUIRED}" | sed "s#'#'\"'\"'#g;s#.*#par_tmp_space_required='&'#" ; else echo "# par_tmp_space_required="; fi ) +$( if [ ! -z ${VIASH_PAR_PUBLISH_SPACE_REQUIRED+x} ]; then echo "${VIASH_PAR_PUBLISH_SPACE_REQUIRED}" | sed "s#'#'\"'\"'#g;s#.*#par_publish_space_required='&'#" ; else echo "# par_publish_space_required="; fi ) +$( if [ ! -z ${VIASH_PAR_PUBLISH_DIR+x} ]; then echo "${VIASH_PAR_PUBLISH_DIR}" | sed "s#'#'\"'\"'#g;s#.*#par_publish_dir='&'#" ; else echo "# par_publish_dir="; fi ) $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\"'\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\"'\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) @@ -1058,12 +1078,24 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\"'\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) ## VIASH END -usage_percentage=\$(df -h "\$(pwd)" | awk 'NR==2 {print \$5}' | tr -d '%') -df -h "\$(pwd)" > "\$par_output" +#!/bin/bash -max_allowed_usage=\$((100-par_req_disk_space)) -if [ "\$usage_percentage" -gt "\$max_allowed_usage" ]; then - echo "WARNING: Disk usage is at \${usage_percentage}%, exceeding maximum allowed usage of \${max_allowed_usage}%" +# Check temporary directory space +tmp_avail_kb=\$(df -k "\$meta_temp_dir" | awk 'NR==2 {print \$4}') +tmp_avail_mb=\$((tmp_avail_kb / 1024)) +echo -e "\\nTemporary directory (\$meta_temp_dir) available space: \${tmp_avail_mb}MB" >> "\$par_output" + +if [ "\$tmp_avail_mb" -lt "\$par_tmp_space_required" ]; then + echo "WARNING: Available temporary space (\${tmp_avail_mb}MB) is less than required (\${par_tmp_space_required}MB)" | tee -a "\$par_output" +fi + +# Check publish directory space if specified +publish_avail_kb=\$(df -k "\$par_publish_dir" | awk 'NR==2 {print \$4}') +publish_avail_mb=\$((publish_avail_kb / 1024)) +echo -e "\\nPublish directory (\$par_publish_dir) available space: \${publish_avail_mb}MB" >> "\$par_output" + +if [ "\$publish_avail_mb" -lt "\$par_publish_space_required" ]; then + echo "WARNING: Available publish space (\${publish_avail_mb}MB) is less than required (\${par_publish_space_required}MB)" | tee -a "\$par_output" fi VIASHMAIN bash "\$tempscript" & @@ -1075,8 +1107,8 @@ VIASHEOF if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # strip viash automount from file paths - if [ ! -z "$VIASH_PAR_INPUT" ]; then - VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + if [ ! -z "$VIASH_PAR_PUBLISH_DIR" ]; then + VIASH_PAR_PUBLISH_DIR=$(ViashDockerStripAutomount "$VIASH_PAR_PUBLISH_DIR") fi if [ ! -z "$VIASH_PAR_OUTPUT" ]; then VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") diff --git a/target/nextflow/check_resources/.config.vsh.yaml b/target/nextflow/check_resources/.config.vsh.yaml index 036ed02..b480a39 100644 --- a/target/nextflow/check_resources/.config.vsh.yaml +++ b/target/nextflow/check_resources/.config.vsh.yaml @@ -3,27 +3,33 @@ version: "check-resources" argument_groups: - name: "Inputs" arguments: - - type: "file" - name: "--input" - description: "Input yaml file containing input parameters of a workflow." + - type: "integer" + name: "--tmp_space_required" + description: "Temporary space required in MB." info: null default: - - "input.yaml" - must_exist: true - create_parent: true - required: true + - 1000 + required: false direction: "input" multiple: false multiple_sep: ";" -- name: "Options" - arguments: - type: "integer" - name: "--req_disk_space" - description: "Required disk space percentage." + name: "--publish_space_required" + description: "Publish space required in MB." info: null default: - - 20 - required: true + - 500 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--publish_dir" + description: "Directory where workflow outputs will be published." + info: null + must_exist: true + create_parent: true + required: false direction: "input" multiple: false multiple_sep: ";" @@ -37,7 +43,7 @@ argument_groups: - "output.txt" must_exist: true create_parent: true - required: true + required: false direction: "output" multiple: false multiple_sep: ";" @@ -46,6 +52,10 @@ resources: path: "script.sh" is_executable: true description: "Check for available resources on the system." +test_resources: +- type: "bash_script" + path: "test.sh" + is_executable: true info: null status: "enabled" requirements: @@ -137,7 +147,7 @@ build_info: output: "target/nextflow/check_resources" executable: "target/nextflow/check_resources/main.nf" viash_version: "0.9.0" - git_commit: "bd253a050d688b378752759d8c5293f4e333ee34" + git_commit: "cf2c8d1810fd106dbf1c36a5d25e885f8285d541" git_remote: "https://github.com/viash-hub/toolbox" package_config: name: "toolbox" diff --git a/target/nextflow/check_resources/main.nf b/target/nextflow/check_resources/main.nf index 4b104b9..56d96c6 100644 --- a/target/nextflow/check_resources/main.nf +++ b/target/nextflow/check_resources/main.nf @@ -2811,32 +2811,36 @@ meta = [ "name" : "Inputs", "arguments" : [ { - "type" : "file", - "name" : "--input", - "description" : "Input yaml file containing input parameters of a workflow.", + "type" : "integer", + "name" : "--tmp_space_required", + "description" : "Temporary space required in MB.", "default" : [ - "input.yaml" + 1000 ], - "must_exist" : true, - "create_parent" : true, - "required" : true, + "required" : false, "direction" : "input", "multiple" : false, "multiple_sep" : ";" - } - ] - }, - { - "name" : "Options", - "arguments" : [ + }, { "type" : "integer", - "name" : "--req_disk_space", - "description" : "Required disk space percentage.", + "name" : "--publish_space_required", + "description" : "Publish space required in MB.", "default" : [ - 20 + 500 ], - "required" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--publish_dir", + "description" : "Directory where workflow outputs will be published.", + "must_exist" : true, + "create_parent" : true, + "required" : false, "direction" : "input", "multiple" : false, "multiple_sep" : ";" @@ -2855,7 +2859,7 @@ meta = [ ], "must_exist" : true, "create_parent" : true, - "required" : true, + "required" : false, "direction" : "output", "multiple" : false, "multiple_sep" : ";" @@ -2871,6 +2875,13 @@ meta = [ } ], "description" : "Check for available resources on the system.", + "test_resources" : [ + { + "type" : "bash_script", + "path" : "test.sh", + "is_executable" : true + } + ], "status" : "enabled", "requirements" : { "commands" : [ @@ -2975,7 +2986,7 @@ meta = [ "engine" : "docker|native", "output" : "target/nextflow/check_resources", "viash_version" : "0.9.0", - "git_commit" : "bd253a050d688b378752759d8c5293f4e333ee34", + "git_commit" : "cf2c8d1810fd106dbf1c36a5d25e885f8285d541", "git_remote" : "https://github.com/viash-hub/toolbox" }, "package_config" : { @@ -3017,8 +3028,9 @@ tempscript=".viash_script.sh" cat > "$tempscript" << VIASHMAIN ## VIASH START # The following code has been auto-generated by Viash. -$( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "${VIASH_PAR_INPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_input='&'#" ; else echo "# par_input="; fi ) -$( if [ ! -z ${VIASH_PAR_REQ_DISK_SPACE+x} ]; then echo "${VIASH_PAR_REQ_DISK_SPACE}" | sed "s#'#'\\"'\\"'#g;s#.*#par_req_disk_space='&'#" ; else echo "# par_req_disk_space="; fi ) +$( if [ ! -z ${VIASH_PAR_TMP_SPACE_REQUIRED+x} ]; then echo "${VIASH_PAR_TMP_SPACE_REQUIRED}" | sed "s#'#'\\"'\\"'#g;s#.*#par_tmp_space_required='&'#" ; else echo "# par_tmp_space_required="; fi ) +$( if [ ! -z ${VIASH_PAR_PUBLISH_SPACE_REQUIRED+x} ]; then echo "${VIASH_PAR_PUBLISH_SPACE_REQUIRED}" | sed "s#'#'\\"'\\"'#g;s#.*#par_publish_space_required='&'#" ; else echo "# par_publish_space_required="; fi ) +$( if [ ! -z ${VIASH_PAR_PUBLISH_DIR+x} ]; then echo "${VIASH_PAR_PUBLISH_DIR}" | sed "s#'#'\\"'\\"'#g;s#.*#par_publish_dir='&'#" ; else echo "# par_publish_dir="; fi ) $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "${VIASH_PAR_OUTPUT}" | sed "s#'#'\\"'\\"'#g;s#.*#par_output='&'#" ; else echo "# par_output="; fi ) $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "${VIASH_META_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_name='&'#" ; else echo "# meta_name="; fi ) $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "${VIASH_META_FUNCTIONALITY_NAME}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_functionality_name='&'#" ; else echo "# meta_functionality_name="; fi ) @@ -3040,12 +3052,24 @@ $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "${VIASH_META_MEMORY_TIB}" $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "${VIASH_META_MEMORY_PIB}" | sed "s#'#'\\"'\\"'#g;s#.*#meta_memory_pib='&'#" ; else echo "# meta_memory_pib="; fi ) ## VIASH END -usage_percentage=\\$(df -h "\\$(pwd)" | awk 'NR==2 {print \\$5}' | tr -d '%') -df -h "\\$(pwd)" > "\\$par_output" +#!/bin/bash -max_allowed_usage=\\$((100-par_req_disk_space)) -if [ "\\$usage_percentage" -gt "\\$max_allowed_usage" ]; then - echo "WARNING: Disk usage is at \\${usage_percentage}%, exceeding maximum allowed usage of \\${max_allowed_usage}%" +# Check temporary directory space +tmp_avail_kb=\\$(df -k "\\$meta_temp_dir" | awk 'NR==2 {print \\$4}') +tmp_avail_mb=\\$((tmp_avail_kb / 1024)) +echo -e "\\\\nTemporary directory (\\$meta_temp_dir) available space: \\${tmp_avail_mb}MB" >> "\\$par_output" + +if [ "\\$tmp_avail_mb" -lt "\\$par_tmp_space_required" ]; then + echo "WARNING: Available temporary space (\\${tmp_avail_mb}MB) is less than required (\\${par_tmp_space_required}MB)" | tee -a "\\$par_output" +fi + +# Check publish directory space if specified +publish_avail_kb=\\$(df -k "\\$par_publish_dir" | awk 'NR==2 {print \\$4}') +publish_avail_mb=\\$((publish_avail_kb / 1024)) +echo -e "\\\\nPublish directory (\\$par_publish_dir) available space: \\${publish_avail_mb}MB" >> "\\$par_output" + +if [ "\\$publish_avail_mb" -lt "\\$par_publish_space_required" ]; then + echo "WARNING: Available publish space (\\${publish_avail_mb}MB) is less than required (\\${par_publish_space_required}MB)" | tee -a "\\$par_output" fi VIASHMAIN bash "$tempscript" diff --git a/target/nextflow/check_resources/nextflow_schema.json b/target/nextflow/check_resources/nextflow_schema.json index 5cd1282..5b52cca 100644 --- a/target/nextflow/check_resources/nextflow_schema.json +++ b/target/nextflow/check_resources/nextflow_schema.json @@ -1,121 +1,70 @@ { -"$schema": "http://json-schema.org/draft-07/schema", -"title": "check_resources", -"description": "Check for available resources on the system.", -"type": "object", -"definitions": { - - - - "inputs" : { - "title": "Inputs", - "type": "object", - "description": "No description", - "properties": { - - - "input": { - "type": - "string", - "description": "Type: `file`, required, default: `input.yaml`. Input yaml file containing input parameters of a workflow", - "help_text": "Type: `file`, required, default: `input.yaml`. Input yaml file containing input parameters of a workflow." - , - "default":"input.yaml" - } - - -} -}, - - - "outputs" : { - "title": "Outputs", - "type": "object", - "description": "No description", - "properties": { - - - "output": { - "type": - "string", - "description": "Type: `file`, required, default: `$id.$key.output.txt`. Output file containing system resources details", - "help_text": "Type: `file`, required, default: `$id.$key.output.txt`. Output file containing system resources details." - , - "default":"$id.$key.output.txt" - } - - -} -}, - - - "options" : { - "title": "Options", - "type": "object", - "description": "No description", - "properties": { - - - "req_disk_space": { - "type": - "integer", - "description": "Type: `integer`, required, default: `20`. Required disk space percentage", - "help_text": "Type: `integer`, required, default: `20`. Required disk space percentage." - , - "default":20 - } - - -} -}, - - - "nextflow input-output arguments" : { - "title": "Nextflow input-output arguments", - "type": "object", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "properties": { - - - "publish_dir": { - "type": - "string", - "description": "Type: `string`, required, example: `output/`. Path to an output directory", - "help_text": "Type: `string`, required, example: `output/`. Path to an output directory." - - } - - - , - "param_list": { - "type": - "string", - "description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel", - "help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.", - "hidden": true - - } - - -} -} -}, -"allOf": [ - - { - "$ref": "#/definitions/inputs" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "check_resources", + "description": "Check for available resources on the system.", + "type": "object", + "$defs": { + "inputs": { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + "tmp_space_required": { + "type": "integer", + "description": "Temporary space required in MB.", + "help_text": "Type: `integer`, multiple: `False`, default: `1000`. ", + "default": 1000 + }, + "publish_space_required": { + "type": "integer", + "description": "Publish space required in MB.", + "help_text": "Type: `integer`, multiple: `False`, default: `500`. ", + "default": 500 + }, + "publish_dir": { + "type": "string", + "format": "path", + "description": "Directory where workflow outputs will be published.", + "help_text": "Type: `file`, multiple: `False`, direction: `input`. " + } + } }, - - { - "$ref": "#/definitions/outputs" + "outputs": { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + "output": { + "type": "string", + "format": "path", + "description": "Output file containing system resources details.", + "help_text": "Type: `file`, multiple: `False`, default: `\"output.txt\"`, direction: `output`. ", + "default": "output.txt" + } + } }, - - { - "$ref": "#/definitions/options" - }, - - { - "$ref": "#/definitions/nextflow input-output arguments" + "nextflow input-output arguments": { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + "publish_dir": { + "type": "string", + "description": "Path to an output directory.", + "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " + } + } } -] + }, + "allOf": [ + { + "$ref": "#/$defs/inputs" + }, + { + "$ref": "#/$defs/outputs" + }, + { + "$ref": "#/$defs/nextflow input-output arguments" + } + ] }